You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/02/15 00:56:38 UTC
[01/50] [abbrv] hadoop git commit: HDFS-11379. DFSInputStream may
infinite loop requesting block locations. Contributed by Daryn Sharp. [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-2915 08dc09581 -> 8d9be8425 (forced update)
HDFS-11379. DFSInputStream may infinite loop requesting block locations. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07a5184f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07a5184f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07a5184f
Branch: refs/heads/YARN-2915
Commit: 07a5184f74fdeffc42cdaec42ad4378c0e41c541
Parents: 2b7a7bb
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Feb 10 12:27:08 2017 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Feb 10 12:27:08 2017 -0600
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSInputStream.java | 48 ++++++++----------
.../java/org/apache/hadoop/hdfs/TestPread.java | 51 ++++++++++++++++++++
2 files changed, 70 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07a5184f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 5783f90..39d0eed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -421,33 +421,36 @@ public class DFSInputStream extends FSInputStream
}
else {
// search cached blocks first
- int targetBlockIdx = locatedBlocks.findBlock(offset);
- if (targetBlockIdx < 0) { // block is not cached
- targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
- // fetch more blocks
- final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
- assert (newBlocks != null) : "Could not find target position " + offset;
- locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
- }
- blk = locatedBlocks.get(targetBlockIdx);
+ blk = fetchBlockAt(offset, 0, true);
}
return blk;
}
}
/** Fetch a block from namenode and cache it */
- protected void fetchBlockAt(long offset) throws IOException {
+ protected LocatedBlock fetchBlockAt(long offset) throws IOException {
+ return fetchBlockAt(offset, 0, false); // don't use cache
+ }
+
+ /** Fetch a block from namenode and cache it */
+ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache)
+ throws IOException {
synchronized(infoLock) {
int targetBlockIdx = locatedBlocks.findBlock(offset);
if (targetBlockIdx < 0) { // block is not cached
targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
+ useCache = false;
}
- // fetch blocks
- final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
- if (newBlocks == null) {
- throw new IOException("Could not find target position " + offset);
+ if (!useCache) { // fetch blocks
+ final LocatedBlocks newBlocks = (length == 0)
+ ? dfsClient.getLocatedBlocks(src, offset)
+ : dfsClient.getLocatedBlocks(src, offset, length);
+ if (newBlocks == null || newBlocks.locatedBlockCount() == 0) {
+ throw new EOFException("Could not find target position " + offset);
+ }
+ locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
}
- locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
+ return locatedBlocks.get(targetBlockIdx);
}
}
@@ -502,28 +505,15 @@ public class DFSInputStream extends FSInputStream
assert (locatedBlocks != null) : "locatedBlocks is null";
List<LocatedBlock> blockRange = new ArrayList<>();
// search cached blocks first
- int blockIdx = locatedBlocks.findBlock(offset);
- if (blockIdx < 0) { // block is not cached
- blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
- }
long remaining = length;
long curOff = offset;
while(remaining > 0) {
- LocatedBlock blk = null;
- if(blockIdx < locatedBlocks.locatedBlockCount())
- blk = locatedBlocks.get(blockIdx);
- if (blk == null || curOff < blk.getStartOffset()) {
- LocatedBlocks newBlocks;
- newBlocks = dfsClient.getLocatedBlocks(src, curOff, remaining);
- locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
- continue;
- }
+ LocatedBlock blk = fetchBlockAt(curOff, remaining, true);
assert curOff >= blk.getStartOffset() : "Block not found";
blockRange.add(blk);
long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
remaining -= bytesRead;
curOff += bytesRead;
- blockIdx++;
}
return blockRange;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07a5184f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index c761225..637f2df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Random;
@@ -28,6 +29,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ChecksumException;
@@ -42,6 +44,7 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -491,6 +494,54 @@ public class TestPread {
}
}
+ @Test
+ public void testTruncateWhileReading() throws Exception {
+ Path path = new Path("/testfile");
+ final int blockSize = 512;
+
+ // prevent initial pre-fetch of multiple block locations
+ Configuration conf = new Configuration();
+ conf.setLong(HdfsClientConfigKeys.Read.PREFETCH_SIZE_KEY, blockSize);
+
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ try {
+ DistributedFileSystem fs = cluster.getFileSystem();
+ // create multi-block file
+ FSDataOutputStream dos =
+ fs.create(path, true, blockSize, (short)1, blockSize);
+ dos.write(new byte[blockSize*3]);
+ dos.close();
+ // truncate a file while it's open
+ final FSDataInputStream dis = fs.open(path);
+ while (!fs.truncate(path, 10)) {
+ Thread.sleep(10);
+ }
+ // verify that reading bytes outside the initial pre-fetch do
+ // not send the client into an infinite loop querying locations.
+ ExecutorService executor = Executors.newFixedThreadPool(1);
+ Future<?> future = executor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws IOException {
+ // read from 2nd block.
+ dis.readFully(blockSize, new byte[4]);
+ return null;
+ }
+ });
+ try {
+ future.get(4, TimeUnit.SECONDS);
+ Assert.fail();
+ } catch (ExecutionException ee) {
+ assertTrue(ee.toString(), ee.getCause() instanceof EOFException);
+ } finally {
+ future.cancel(true);
+ executor.shutdown();
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
public static void main(String[] args) throws Exception {
new TestPread().testPreadDFS();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-3933. FairScheduler: Multiple
calls to completedContainer are not safe. (Shiwei Guo and Miklos Szegedi via
kasha)
Posted by su...@apache.org.
YARN-3933. FairScheduler: Multiple calls to completedContainer are not safe. (Shiwei Guo and Miklos Szegedi via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/646c6d65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/646c6d65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/646c6d65
Branch: refs/heads/YARN-2915
Commit: 646c6d6509f515b1373288869fb92807fa2ddc9b
Parents: cc45da7
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Feb 13 11:26:30 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Feb 13 11:26:30 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSAppAttempt.java | 9 +++-
.../scheduler/fair/TestFairScheduler.java | 55 +++++++++++++++++++-
2 files changed, 61 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/646c6d65/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 9e57fa7..6dfcc84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -140,6 +140,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Container container = rmContainer.getContainer();
ContainerId containerId = container.getId();
+ // Remove from the list of containers
+ if (liveContainers.remove(containerId) == null) {
+ LOG.info("Additional complete request on completed container " +
+ rmContainer.getContainerId());
+ return;
+ }
+
// Remove from the list of newly allocated containers if found
newlyAllocatedContainers.remove(rmContainer);
@@ -151,8 +158,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
+ " in state: " + rmContainer.getState() + " event:" + event);
}
- // Remove from the list of containers
- liveContainers.remove(rmContainer.getContainerId());
untrackContainerForPreemption(rmContainer);
Resource containerResource = rmContainer.getContainer().getResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/646c6d65/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index ce32459..da5d3ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -99,6 +100,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -3552,7 +3554,58 @@ public class TestFairScheduler extends FairSchedulerTestBase {
verifyAppRunnable(attId5, false);
verifyQueueNumRunnable("queue1", 2, 1);
}
-
+
+ @Test
+ public void testMultipleCompletedEvent() throws Exception {
+ // Set up a fair scheduler
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println("<maxAMShare>0.2</maxAMShare>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Create a node
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(20480, 20),
+ 0, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.update();
+
+ // Launch an app
+ ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
+ createApplicationWithAMResource(
+ attId1, "queue1", "user1",
+ Resource.newInstance(1024, 1));
+ createSchedulingRequestExistingApplication(
+ 1024, 1,
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(), attId1);
+ FSAppAttempt app1 = scheduler.getSchedulerApp(attId1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ RMContainer container = app1.getLiveContainersMap().
+ values().iterator().next();
+ scheduler.completedContainer(container, SchedulerUtils
+ .createAbnormalContainerStatus(container.getContainerId(),
+ SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL);
+ scheduler.completedContainer(container, SchedulerUtils
+ .createAbnormalContainerStatus(container.getContainerId(),
+ SchedulerUtils.COMPLETED_APPLICATION),
+ RMContainerEventType.FINISHED);
+ assertEquals(Resources.none(), app1.getResourceUsage());
+ }
+
@Test
public void testQueueMaxAMShare() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-5390. Federation Subcluster
Resolver. Contributed by Ellen Hui.
Posted by su...@apache.org.
YARN-5390. Federation Subcluster Resolver. Contributed by Ellen Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e5e94bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e5e94bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e5e94bd
Branch: refs/heads/YARN-2915
Commit: 9e5e94bdc066a583322cab5f362707c845be8563
Parents: bbb8884
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Aug 4 15:58:31 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../src/main/resources/yarn-default.xml | 7 +
.../hadoop-yarn-server-common/pom.xml | 10 +
.../resolver/AbstractSubClusterResolver.java | 67 +++++++
.../resolver/DefaultSubClusterResolverImpl.java | 164 +++++++++++++++++
.../federation/resolver/SubClusterResolver.java | 58 ++++++
.../federation/resolver/package-info.java | 17 ++
.../resolver/TestDefaultSubClusterResolver.java | 184 +++++++++++++++++++
.../src/test/resources/nodes | 4 +
.../src/test/resources/nodes-malformed | 3 +
10 files changed, 522 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 136227a..a047450 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2482,6 +2482,14 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20;
////////////////////////////////
+ // Federation Configs
+ ////////////////////////////////
+
+ public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+ public static final String FEDERATION_MACHINE_LIST =
+ FEDERATION_PREFIX + "machine-list";
+
+ ////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4ca46f9..31c86f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2611,6 +2611,13 @@
<!-- Other Configuration -->
<property>
+ <description>
+ Machine list file to be loaded by the FederationSubCluster Resolver
+ </description>
+ <name>yarn.federation.machine-list</name>
+ </property>
+
+ <property>
<description>The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 9cc3cae..6d2fbef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -177,6 +177,16 @@
</filesets>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/nodes</exclude>
+ <exclude>src/test/resources/nodes-malformed</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
new file mode 100644
index 0000000..8238633
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
@@ -0,0 +1,67 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Map;
+
+/**
+ * Partial implementation of {@link SubClusterResolver}, containing basic
+ * implementations of the read methods.
+ */
+public abstract class AbstractSubClusterResolver implements SubClusterResolver {
+ private Map<String, SubClusterId> nodeToSubCluster =
+ new HashMap<String, SubClusterId>();
+ private Map<String, Set<SubClusterId>> rackToSubClusters =
+ new HashMap<String, Set<SubClusterId>>();
+
+ @Override
+ public SubClusterId getSubClusterForNode(String nodename)
+ throws YarnException {
+ SubClusterId subClusterId = this.nodeToSubCluster.get(nodename);
+
+ if (subClusterId == null) {
+ throw new YarnException("Cannot find subClusterId for node " + nodename);
+ }
+
+ return subClusterId;
+ }
+
+ @Override
+ public Set<SubClusterId> getSubClustersForRack(String rackname)
+ throws YarnException {
+ if (!rackToSubClusters.containsKey(rackname)) {
+ throw new YarnException("Cannot resolve rack " + rackname);
+ }
+
+ return rackToSubClusters.get(rackname);
+ }
+
+ protected Map<String, SubClusterId> getNodeToSubCluster() {
+ return nodeToSubCluster;
+ }
+
+ protected Map<String, Set<SubClusterId>> getRackToSubClusters() {
+ return rackToSubClusters;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java
new file mode 100644
index 0000000..d3c5c26
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java
@@ -0,0 +1,164 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import java.io.BufferedReader;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Default simple sub-cluster and rack resolver class.
+ *
+ * This class expects a three-column comma separated file, specified in
+ * yarn.federation.machine-list. Each line of the file should be of the format:
+ *
+ * nodeName, subClusterId, rackName
+ *
+ * Lines that do not follow this format will be ignored. This resolver only
+ * loads the file when load() is explicitly called; it will not react to changes
+ * to the file.
+ *
+ * It is case-insensitive on the rack and node names and ignores
+ * leading/trailing whitespace.
+ *
+ */
+public class DefaultSubClusterResolverImpl extends AbstractSubClusterResolver
+ implements SubClusterResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultSubClusterResolverImpl.class);
+ private Configuration conf;
+
+ // Index of the node hostname in the machine info file.
+ private static final int NODE_NAME_INDEX = 0;
+
+ // Index of the sub-cluster ID in the machine info file.
+ private static final int SUBCLUSTER_ID_INDEX = 1;
+
+ // Index of the rack name ID in the machine info file.
+ private static final int RACK_NAME_INDEX = 2;
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public SubClusterId getSubClusterForNode(String nodename)
+ throws YarnException {
+ return super.getSubClusterForNode(nodename.toUpperCase());
+ }
+
+ @Override
+ public void load() {
+ String fileName =
+ this.conf.get(YarnConfiguration.FEDERATION_MACHINE_LIST, "");
+
+ try {
+ if (fileName == null || fileName.trim().length() == 0) {
+ LOG.info(
+ "The machine list file path is not specified in the configuration");
+ return;
+ }
+
+ Path file = null;
+ BufferedReader reader = null;
+
+ try {
+ file = Paths.get(fileName);
+ } catch (InvalidPathException e) {
+ LOG.info("The configured machine list file path {} does not exist",
+ fileName);
+ return;
+ }
+
+ try {
+ reader = Files.newBufferedReader(file, Charset.defaultCharset());
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ String[] tokens = line.split(",");
+ if (tokens.length == 3) {
+
+ String nodeName = tokens[NODE_NAME_INDEX].trim().toUpperCase();
+ SubClusterId subClusterId =
+ SubClusterId.newInstance(tokens[SUBCLUSTER_ID_INDEX].trim());
+ String rackName = tokens[RACK_NAME_INDEX].trim().toUpperCase();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading node into resolver: {} --> {}", nodeName,
+ subClusterId);
+ LOG.debug("Loading rack into resolver: {} --> {} ", rackName,
+ subClusterId);
+ }
+
+ this.getNodeToSubCluster().put(nodeName, subClusterId);
+ loadRackToSubCluster(rackName, subClusterId);
+ } else {
+ LOG.warn("Skipping malformed line in machine list: " + line);
+ }
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ LOG.info("Successfully loaded file {}", fileName);
+
+ } catch (Exception e) {
+ LOG.error("Failed to parse file " + fileName, e);
+ }
+ }
+
+ private void loadRackToSubCluster(String rackName,
+ SubClusterId subClusterId) {
+ String rackNameUpper = rackName.toUpperCase();
+
+ if (!this.getRackToSubClusters().containsKey(rackNameUpper)) {
+ this.getRackToSubClusters().put(rackNameUpper,
+ new HashSet<SubClusterId>());
+ }
+
+ this.getRackToSubClusters().get(rackNameUpper).add(subClusterId);
+
+ }
+
+ @Override
+ public Set<SubClusterId> getSubClustersForRack(String rackname)
+ throws YarnException {
+ return super.getSubClustersForRack(rackname.toUpperCase());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java
new file mode 100644
index 0000000..c6adfa6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java
@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * An utility that helps to determine the sub-cluster that a specified node
+ * belongs to.
+ */
+public interface SubClusterResolver extends Configurable {
+
+ /**
+ * Obtain the sub-cluster that a specified node belongs to.
+ *
+ * @param nodename the node whose sub-cluster is to be determined
+ * @return the sub-cluster as identified by the {@link SubClusterId} that the
+ * node belongs to
+ * @throws YarnException if the node's sub-cluster cannot be resolved
+ */
+ SubClusterId getSubClusterForNode(String nodename) throws YarnException;
+
+ /**
+ * Obtain the sub-clusters that have nodes on a specified rack.
+ *
+ * @param rackname the name of the rack
+ * @return the sub-clusters as identified by the {@link SubClusterId} that
+ * have nodes on the given rack
+ * @throws YarnException if the sub-cluster of any node on the rack cannot be
+ * resolved, or if the rack name is not recognized
+ */
+ Set<SubClusterId> getSubClustersForRack(String rackname) throws YarnException;
+
+ /**
+ * Load the nodes to subCluster mapping from the file.
+ */
+ void load();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java
new file mode 100644
index 0000000..c042660
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.resolver;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java
new file mode 100644
index 0000000..7396942
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java
@@ -0,0 +1,184 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test {@link SubClusterResolver} against correct and malformed Federation
+ * machine lists.
+ */
+public class TestDefaultSubClusterResolver {
+ private static YarnConfiguration conf;
+ private static SubClusterResolver resolver;
+
+ public static void setUpGoodFile() {
+ conf = new YarnConfiguration();
+ resolver = new DefaultSubClusterResolverImpl();
+
+ URL url =
+ Thread.currentThread().getContextClassLoader().getResource("nodes");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'nodes' dummy file in classpath");
+ }
+
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, url.getPath());
+ resolver.setConf(conf);
+ resolver.load();
+ }
+
+ private void setUpMalformedFile() {
+ conf = new YarnConfiguration();
+ resolver = new DefaultSubClusterResolverImpl();
+
+ URL url = Thread.currentThread().getContextClassLoader()
+ .getResource("nodes-malformed");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'nodes-malformed' dummy file in classpath");
+ }
+
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, url.getPath());
+ resolver.setConf(conf);
+ resolver.load();
+ }
+
+ private void setUpNonExistentFile() {
+ conf = new YarnConfiguration();
+ resolver = new DefaultSubClusterResolverImpl();
+
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, "fileDoesNotExist");
+ resolver.setConf(conf);
+ resolver.load();
+ }
+
+ @Test
+ public void testGetSubClusterForNode() throws YarnException {
+ setUpGoodFile();
+
+ // All lowercase, no whitespace in machine list file
+ Assert.assertEquals(SubClusterId.newInstance("subcluster1"),
+ resolver.getSubClusterForNode("node1"));
+ // Leading and trailing whitespace in machine list file
+ Assert.assertEquals(SubClusterId.newInstance("subcluster2"),
+ resolver.getSubClusterForNode("node2"));
+ // Node name capitalization in machine list file
+ Assert.assertEquals(SubClusterId.newInstance("subcluster3"),
+ resolver.getSubClusterForNode("node3"));
+
+ try {
+ resolver.getSubClusterForNode("nodeDoesNotExist");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterForNodeMalformedFile() throws YarnException {
+ setUpMalformedFile();
+
+ try {
+ resolver.getSubClusterForNode("node1");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+
+ try {
+ resolver.getSubClusterForNode("node2");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+
+ Assert.assertEquals(SubClusterId.newInstance("subcluster3"),
+ resolver.getSubClusterForNode("node3"));
+
+ try {
+ resolver.getSubClusterForNode("nodeDoesNotExist");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterForNodeNoFile() throws YarnException {
+ setUpNonExistentFile();
+
+ try {
+ resolver.getSubClusterForNode("node1");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+ }
+
+ @Test
+ public void testGetSubClustersForRack() throws YarnException {
+ setUpGoodFile();
+
+ Set<SubClusterId> rack1Expected = new HashSet<SubClusterId>();
+ rack1Expected.add(SubClusterId.newInstance("subcluster1"));
+ rack1Expected.add(SubClusterId.newInstance("subcluster2"));
+
+ Set<SubClusterId> rack2Expected = new HashSet<SubClusterId>();
+ rack2Expected.add(SubClusterId.newInstance("subcluster3"));
+
+ // Two subclusters have nodes in rack1
+ Assert.assertEquals(rack1Expected, resolver.getSubClustersForRack("rack1"));
+
+ // Two nodes are in rack2, but both belong to subcluster3
+ Assert.assertEquals(rack2Expected, resolver.getSubClustersForRack("rack2"));
+
+ try {
+ resolver.getSubClustersForRack("rackDoesNotExist");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Cannot resolve rack"));
+ }
+ }
+
+ @Test
+ public void testGetSubClustersForRackNoFile() throws YarnException {
+ setUpNonExistentFile();
+
+ try {
+ resolver.getSubClustersForRack("rack1");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Cannot resolve rack"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
new file mode 100644
index 0000000..e4d6112
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
@@ -0,0 +1,4 @@
+node1,subcluster1,rack1
+ node2 , subcluster2, RACK1
+noDE3,subcluster3, rack2
+node4, subcluster3, rack2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e5e94bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed
new file mode 100644
index 0000000..6d0aa39
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed
@@ -0,0 +1,3 @@
+node1,
+node2,subcluster2,subCluster2, rack1
+node3,subcluster3, rack2
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-14069. AliyunOSS:
listStatus returns wrong file info. Contributed by Fei Hui
Posted by su...@apache.org.
HADOOP-14069. AliyunOSS: listStatus returns wrong file info. Contributed by Fei Hui
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01be4503
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01be4503
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01be4503
Branch: refs/heads/YARN-2915
Commit: 01be4503c3b053d2cff0b179774dabfd267877db
Parents: 839b690
Author: Kai Zheng <ka...@intel.com>
Authored: Mon Feb 13 13:18:20 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Mon Feb 13 13:18:20 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java | 2 +-
.../hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01be4503/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 3426319..0491087 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -352,7 +352,7 @@ public class AliyunOSSFileSystem extends FileSystem {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding: rd: " + keyPath);
}
- result.add(new FileStatus(0, true, 1, 0, 0, keyPath));
+ result.add(getFileStatus(keyPath));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01be4503/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
index ad8ef6e..3ebf507 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -206,6 +207,12 @@ public class TestAliyunOSSFileSystemContract
assertTrue("Should be directory",
this.fs.getFileStatus(dirPath).isDirectory());
assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile());
+
+ Path parentPath = this.path("/test/oss");
+ for (FileStatus fileStatus: fs.listStatus(parentPath)) {
+ assertTrue("file and directory should be new",
+ fileStatus.getModificationTime() > 0L);
+ }
}
public void testMkdirsForExistingFile() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-5467. InputValidator for the
FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
Posted by su...@apache.org.
YARN-5467. InputValidator for the FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6028f167
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6028f167
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6028f167
Branch: refs/heads/YARN-2915
Commit: 6028f167707e456f0db2f1f6aabff0d369627c3b
Parents: 9ea752f
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Aug 17 12:07:06 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../store/impl/MemoryFederationStateStore.java | 30 +
...cationHomeSubClusterStoreInputValidator.java | 183 +++
...ationMembershipStateStoreInputValidator.java | 317 +++++
.../FederationPolicyStoreInputValidator.java | 144 ++
...derationStateStoreInvalidInputException.java | 48 +
.../federation/store/utils/package-info.java | 17 +
.../impl/FederationStateStoreBaseTest.java | 6 +-
.../TestFederationStateStoreInputValidator.java | 1265 ++++++++++++++++++
8 files changed, 2007 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 8144435..6e564dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -57,6 +57,9 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegister
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
@@ -88,6 +91,8 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
return SubClusterRegisterResponse.newInstance();
@@ -96,6 +101,8 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public SubClusterDeregisterResponse deregisterSubCluster(
SubClusterDeregisterRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
SubClusterInfo subClusterInfo = membership.get(request.getSubClusterId());
if (subClusterInfo == null) {
throw new YarnException(
@@ -111,6 +118,8 @@ public class MemoryFederationStateStore implements FederationStateStore {
public SubClusterHeartbeatResponse subClusterHeartbeat(
SubClusterHeartbeatRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
SubClusterId subClusterId = request.getSubClusterId();
SubClusterInfo subClusterInfo = membership.get(subClusterId);
@@ -129,6 +138,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public GetSubClusterInfoResponse getSubCluster(
GetSubClusterInfoRequest request) throws YarnException {
+
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
SubClusterId subClusterId = request.getSubClusterId();
if (!membership.containsKey(subClusterId)) {
throw new YarnException(
@@ -157,6 +169,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
@@ -172,6 +187,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
if (!applications.containsKey(appId)) {
@@ -186,6 +204,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
throw new YarnException("Application " + appId + " does not exist");
@@ -212,6 +233,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
DeleteApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
throw new YarnException("Application " + appId + " does not exist");
@@ -224,6 +248,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
String queue = request.getQueue();
if (!policies.containsKey(queue)) {
throw new YarnException("Policy for queue " + queue + " does not exist");
@@ -236,6 +263,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
policies.put(request.getPolicyConfiguration().getQueue(),
request.getPolicyConfiguration());
return SetSubClusterPolicyConfigurationResponse.newInstance();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java
new file mode 100644
index 0000000..c14a452
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to
+ * {@code FederationApplicationHomeSubClusterStore}, allows a fail fast
+ * mechanism for invalid user inputs.
+ *
+ */
+public final class FederationApplicationHomeSubClusterStoreInputValidator {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(FederationApplicationHomeSubClusterStoreInputValidator.class);
+
+ private FederationApplicationHomeSubClusterStoreInputValidator() {
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link AddApplicationHomeSubClusterRequest}
+ * for adding a new application is valid or not.
+ *
+ * @param request the {@link AddApplicationHomeSubClusterRequest} to validate
+ * against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateAddApplicationHomeSubClusterRequest(
+ AddApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing AddApplicationHomeSubCluster Request."
+ + " Please try again by specifying"
+ + " an AddApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate ApplicationHomeSubCluster info
+ checkApplicationHomeSubCluster(request.getApplicationHomeSubCluster());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link UpdateApplicationHomeSubClusterRequest}
+ * for updating an application is valid or not.
+ *
+ * @param request the {@link UpdateApplicationHomeSubClusterRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateUpdateApplicationHomeSubClusterRequest(
+ UpdateApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing UpdateApplicationHomeSubCluster Request."
+ + " Please try again by specifying"
+ + " an ApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate ApplicationHomeSubCluster info
+ checkApplicationHomeSubCluster(request.getApplicationHomeSubCluster());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link GetApplicationHomeSubClusterRequest}
+ * for querying application's information is valid or not.
+ *
+ * @param request the {@link GetApplicationHomeSubClusterRequest} to validate
+ * against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateGetApplicationHomeSubClusterRequest(
+ GetApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing GetApplicationHomeSubCluster Request."
+ + " Please try again by specifying an Application Id information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate application Id
+ checkApplicationId(request.getApplicationId());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link DeleteApplicationHomeSubClusterRequest}
+ * for deleting an application is valid or not.
+ *
+ * @param request the {@link DeleteApplicationHomeSubClusterRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateDeleteApplicationHomeSubClusterRequest(
+ DeleteApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing DeleteApplicationHomeSubCluster Request."
+ + " Please try again by specifying"
+ + " an ApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate application Id
+ checkApplicationId(request.getApplicationId());
+ }
+
+ /**
+ * Validate if the ApplicationHomeSubCluster info are present or not.
+ *
+ * @param applicationHomeSubCluster the information of the application to be
+ * verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster Info
+ * are invalid
+ */
+ private static void checkApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster)
+
+ throws FederationStateStoreInvalidInputException {
+ if (applicationHomeSubCluster == null) {
+ String message = "Missing ApplicationHomeSubCluster Info."
+ + " Please try again by specifying"
+ + " an ApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ // validate application Id
+ checkApplicationId(applicationHomeSubCluster.getApplicationId());
+
+ // validate subcluster Id
+ FederationMembershipStateStoreInputValidator
+ .checkSubClusterId(applicationHomeSubCluster.getHomeSubCluster());
+
+ }
+
+ /**
+ * Validate if the application id is present or not.
+ *
+ * @param appId the id of the application to be verified
+ * @throws FederationStateStoreInvalidInputException if the application Id is
+ * invalid
+ */
+ private static void checkApplicationId(ApplicationId appId)
+ throws FederationStateStoreInvalidInputException {
+ if (appId == null) {
+ String message = "Missing Application Id."
+ + " Please try again by specifying an Application Id.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
new file mode 100644
index 0000000..b587ee5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import java.net.URI;
+
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to
+ * {@code FederationMembershipStateStore}, allows a fail fast mechanism for
+ * invalid user inputs.
+ *
+ */
+public final class FederationMembershipStateStoreInputValidator {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(FederationMembershipStateStoreInputValidator.class);
+
+ private FederationMembershipStateStoreInputValidator() {
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link SubClusterRegisterRequest} for
+ * registration a new subcluster is valid or not.
+ *
+ * @param request the {@link SubClusterRegisterRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSubClusterRegisterRequest(
+ SubClusterRegisterRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing SubClusterRegister Request."
+ + " Please try again by specifying a"
+ + " SubCluster Register Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+
+ }
+
+ // validate subcluster info
+ checkSubClusterInfo(request.getSubClusterInfo());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link SubClusterDeregisterRequest} for
+ * deregistration a subcluster is valid or not.
+ *
+ * @param request the {@link SubClusterDeregisterRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSubClusterDeregisterRequest(
+ SubClusterDeregisterRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing SubClusterDeregister Request."
+ + " Please try again by specifying a"
+ + " SubCluster Deregister Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(request.getSubClusterId());
+ // validate subcluster state
+ checkSubClusterState(request.getState());
+ if (!request.getState().isFinal()) {
+ String message = "Invalid non-final state: " + request.getState();
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link SubClusterHeartbeatRequest} for
+ * heartbeating a subcluster is valid or not.
+ *
+ * @param request the {@link SubClusterHeartbeatRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSubClusterHeartbeatRequest(
+ SubClusterHeartbeatRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing SubClusterHeartbeat Request."
+ + " Please try again by specifying a"
+ + " SubCluster Heartbeat Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(request.getSubClusterId());
+ // validate last heartbeat timestamp
+ checkTimestamp(request.getLastHeartBeat());
+ // validate subcluster capability
+ checkCapability(request.getCapability());
+ // validate subcluster state
+ checkSubClusterState(request.getState());
+
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link GetSubClusterInfoRequest} for querying
+ * subcluster's information is valid or not.
+ *
+ * @param request the {@link GetSubClusterInfoRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateGetSubClusterInfoRequest(
+ GetSubClusterInfoRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing GetSubClusterInfo Request."
+ + " Please try again by specifying a Get SubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(request.getSubClusterId());
+ }
+
+ /**
+ * Validate if the SubCluster Info are present or not.
+ *
+ * @param subClusterInfo the information of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster Info
+ * are invalid
+ */
+ private static void checkSubClusterInfo(SubClusterInfo subClusterInfo)
+ throws FederationStateStoreInvalidInputException {
+ if (subClusterInfo == null) {
+ String message = "Missing SubCluster Information."
+ + " Please try again by specifying SubCluster Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(subClusterInfo.getSubClusterId());
+
+ // validate AMRM Service address
+ checkAddress(subClusterInfo.getAMRMServiceAddress());
+ // validate ClientRM Service address
+ checkAddress(subClusterInfo.getClientRMServiceAddress());
+ // validate RMClient Service address
+ checkAddress(subClusterInfo.getRMAdminServiceAddress());
+ // validate RMWeb Service address
+ checkAddress(subClusterInfo.getRMWebServiceAddress());
+
+ // validate last heartbeat timestamp
+ checkTimestamp(subClusterInfo.getLastHeartBeat());
+ // validate last start timestamp
+ checkTimestamp(subClusterInfo.getLastStartTime());
+
+ // validate subcluster state
+ checkSubClusterState(subClusterInfo.getState());
+
+ // validate subcluster capability
+ checkCapability(subClusterInfo.getCapability());
+ }
+
+ /**
+ * Validate if the timestamp is positive or not.
+ *
+ * @param timestamp the timestamp to be verified
+ * @throws FederationStateStoreInvalidInputException if the timestamp is
+ * invalid
+ */
+ private static void checkTimestamp(long timestamp)
+ throws FederationStateStoreInvalidInputException {
+ if (timestamp < 0) {
+ String message = "Invalid timestamp information."
+ + " Please try again by specifying valid Timestamp Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the Capability is present or not.
+ *
+ * @param capability the capability of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the capability is
+ * invalid
+ */
+ private static void checkCapability(String capability)
+ throws FederationStateStoreInvalidInputException {
+ if (capability == null || capability.isEmpty()) {
+ String message = "Invalid capability information."
+ + " Please try again by specifying valid Capability Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the SubCluster Id is present or not.
+ *
+ * @param subClusterId the identifier of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster Id is
+ * invalid
+ */
+ protected static void checkSubClusterId(SubClusterId subClusterId)
+ throws FederationStateStoreInvalidInputException {
+ // check if cluster id is present
+ if (subClusterId == null) {
+ String message = "Missing SubCluster Id information."
+ + " Please try again by specifying Subcluster Id information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ // check if cluster id is valid
+ if (subClusterId.getId().isEmpty()) {
+ String message = "Invalid SubCluster Id information."
+ + " Please try again by specifying valid Subcluster Id.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the SubCluster Address is a valid URL or not.
+ *
+ * @param address the endpoint of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the address is invalid
+ */
+ private static void checkAddress(String address)
+ throws FederationStateStoreInvalidInputException {
+ // Ensure url is not null
+ if (address == null || address.isEmpty()) {
+ String message = "Missing SubCluster Endpoint information."
+ + " Please try again by specifying SubCluster Endpoint information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ // Validate url is well formed
+ boolean hasScheme = address.contains("://");
+ URI uri = null;
+ try {
+ uri = hasScheme ? URI.create(address)
+ : URI.create("dummyscheme://" + address);
+ } catch (IllegalArgumentException e) {
+ String message = "The provided SubCluster Endpoint does not contain a"
+ + " valid host:port authority: " + address;
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ String host = uri.getHost();
+ int port = uri.getPort();
+ String path = uri.getPath();
+ if ((host == null) || (port < 0)
+ || (!hasScheme && path != null && !path.isEmpty())) {
+ String message = "The provided SubCluster Endpoint does not contain a"
+ + " valid host:port authority: " + address;
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the SubCluster State is present or not.
+ *
+ * @param state the state of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster State
+ * is invalid
+ */
+ private static void checkSubClusterState(SubClusterState state)
+ throws FederationStateStoreInvalidInputException {
+ // check sub-cluster state is not empty
+ if (state == null) {
+ String message = "Missing SubCluster State information."
+ + " Please try again by specifying SubCluster State information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java
new file mode 100644
index 0000000..273a8ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to {@code FederationPolicyStore}, allows
+ * a fail fast mechanism for invalid user inputs.
+ *
+ */
+public final class FederationPolicyStoreInputValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationPolicyStoreInputValidator.class);
+
+ private FederationPolicyStoreInputValidator() {
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided
+ * {@link GetSubClusterPolicyConfigurationRequest} for querying policy's
+ * information is valid or not.
+ *
+ * @param request the {@link GetSubClusterPolicyConfigurationRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateGetSubClusterPolicyConfigurationRequest(
+ GetSubClusterPolicyConfigurationRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing GetSubClusterPolicyConfiguration Request."
+ + " Please try again by specifying a policy selection information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate queue id
+ checkQueue(request.getQueue());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided
+ * {@link SetSubClusterPolicyConfigurationRequest} for adding a new policy is
+ * valid or not.
+ *
+ * @param request the {@link SetSubClusterPolicyConfigurationRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSetSubClusterPolicyConfigurationRequest(
+ SetSubClusterPolicyConfigurationRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing SetSubClusterPolicyConfiguration Request."
+ + " Please try again by specifying an policy insertion information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster policy configuration
+ checkSubClusterPolicyConfiguration(request.getPolicyConfiguration());
+ }
+
+ /**
+ * Validate if the SubClusterPolicyConfiguration is valid or not.
+ *
+ * @param policyConfiguration the policy information to be verified
+ * @throws FederationStateStoreInvalidInputException if the policy information
+ * are invalid
+ */
+ private static void checkSubClusterPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration)
+ throws FederationStateStoreInvalidInputException {
+ if (policyConfiguration == null) {
+ String message = "Missing SubClusterPolicyConfiguration."
+ + " Please try again by specifying a SubClusterPolicyConfiguration.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate queue id
+ checkQueue(policyConfiguration.getQueue());
+ // validate policy type
+ checkType(policyConfiguration.getType());
+
+ }
+
+ /**
+ * Validate if the queue id is a valid or not.
+ *
+ * @param queue the queue id of the policy to be verified
+ * @throws FederationStateStoreInvalidInputException if the queue id is
+ * invalid
+ */
+ private static void checkQueue(String queue)
+ throws FederationStateStoreInvalidInputException {
+ if (queue == null || queue.isEmpty()) {
+ String message = "Missing Queue. Please try again by specifying a Queue.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the policy type is a valid or not.
+ *
+ * @param type the type of the policy to be verified
+ * @throws FederationStateStoreInvalidInputException if the policy is invalid
+ */
+ private static void checkType(String type)
+ throws FederationStateStoreInvalidInputException {
+ if (type == null || type.isEmpty()) {
+ String message = "Missing Policy Type."
+ + " Please try again by specifying a Policy Type.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java
new file mode 100644
index 0000000..ea1428d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Exception thrown by the {@link FederationMembershipStateStoreInputValidator},
+ * {@link FederationApplicationHomeSubClusterStoreInputValidator},
+ * {@link FederationPolicyStoreInputValidator} if the input is invalid.
+ *
+ */
+public class FederationStateStoreInvalidInputException extends YarnException {
+
+ /**
+ * IDE auto-generated.
+ */
+ private static final long serialVersionUID = -7352144682711430801L;
+
+ public FederationStateStoreInvalidInputException(Throwable cause) {
+ super(cause);
+ }
+
+ public FederationStateStoreInvalidInputException(String message) {
+ super(message);
+ }
+
+ public FederationStateStoreInvalidInputException(String message,
+ Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java
new file mode 100644
index 0000000..f4a9c7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.utils;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 414696b..63a5b65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -162,9 +162,9 @@ public abstract class FederationStateStoreBaseTest {
SubClusterRegisterRequest.newInstance(subClusterInfo2));
stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
- .newInstance(subClusterId1, SubClusterState.SC_RUNNING, ""));
- stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
- .newInstance(subClusterId2, SubClusterState.SC_UNHEALTHY, ""));
+ .newInstance(subClusterId1, SubClusterState.SC_RUNNING, "capability"));
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest.newInstance(
+ subClusterId2, SubClusterState.SC_UNHEALTHY, "capability"));
Assert.assertTrue(
stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(true))
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-14058. Fix
NativeS3FileSystemContractBaseTest#testDirWithDifferentMarkersWorks.
Contributed by Yiqun Lin.
Posted by su...@apache.org.
HADOOP-14058. Fix NativeS3FileSystemContractBaseTest#testDirWithDifferentMarkersWorks. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9f84912
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9f84912
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9f84912
Branch: refs/heads/YARN-2915
Commit: b9f8491252f5a23a91a1d695d748556a0fd803ae
Parents: aaf106f
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Feb 15 01:45:56 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Feb 15 01:45:56 2017 +0900
----------------------------------------------------------------------
.../hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f84912/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
index ef223ac..261f79b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java
@@ -85,7 +85,7 @@ public abstract class NativeS3FileSystemContractBaseTest
public void testDirWithDifferentMarkersWorks() throws Exception {
- for (int i = 0; i < 3; i++) {
+ for (int i = 0; i <= 3; i++) {
String base = "test/hadoop" + i;
Path path = path("/" + base);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-11084. Add a regression test
for sticky bit support of OIV ReverseXML processor. Contributed by Wei-Chiu
Chuang.
Posted by su...@apache.org.
HDFS-11084. Add a regression test for sticky bit support of OIV ReverseXML processor. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cf59937
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cf59937
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cf59937
Branch: refs/heads/YARN-2915
Commit: 0cf5993712a01993bd701bd9664e6af284378b55
Parents: 1fa084c
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Feb 14 08:59:12 2017 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Feb 14 09:11:55 2017 -0800
----------------------------------------------------------------------
.../tools/offlineImageViewer/TestOfflineImageViewer.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf59937/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 740a8ab..dacbb85 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -69,6 +69,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -158,6 +160,15 @@ public class TestOfflineImageViewer {
hdfs.mkdirs(invalidXMLDir);
dirCount++;
+ //Create a directory with sticky bits
+ Path stickyBitDir = new Path("/stickyBit");
+ hdfs.mkdirs(stickyBitDir);
+ hdfs.setPermission(stickyBitDir, new FsPermission(FsAction.ALL,
+ FsAction.ALL, FsAction.ALL, true));
+ dirCount++;
+ writtenFiles.put(stickyBitDir.toString(),
+ hdfs.getFileStatus(stickyBitDir));
+
// Get delegation tokens so we log the delegation token op
Token<?>[] delegationTokens = hdfs
.addDelegationTokens(TEST_RENEWER, null);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-11408. The config name of
balance bandwidth is out of date. Contributed by Yiqun Lin.
Posted by su...@apache.org.
HDFS-11408. The config name of balance bandwidth is out of date. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4164a203
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4164a203
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4164a203
Branch: refs/heads/YARN-2915
Commit: 4164a2032a41e7318749efd0301751eb2b369cdc
Parents: 719df99
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Feb 14 18:57:20 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Feb 14 18:57:20 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java | 2 +-
.../main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java | 2 +-
.../hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java | 2 +-
.../apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java | 2 +-
.../hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java | 2 +-
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +-
9 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 7b6a4e5..e0ccd62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2233,7 +2233,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/**
* Requests the namenode to tell all datanodes to use a new, non-persistent
- * bandwidth value for dfs.balance.bandwidthPerSec.
+ * bandwidth value for dfs.datanode.balance.bandwidthPerSec.
* See {@link ClientProtocol#setBalancerBandwidth(long)}
* for more details.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index e9475d8..30dcfa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1599,7 +1599,7 @@ public class DistributedFileSystem extends FileSystem {
/**
* Requests the namenode to tell all datanodes to use a new, non-persistent
- * bandwidth value for dfs.balance.bandwidthPerSec.
+ * bandwidth value for dfs.datanode.balance.bandwidthPerSec.
* The bandwidth parameter is the max number of bytes per second of network
* bandwidth to be used by a datanode during balancing.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 407621b..eaebd6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -938,7 +938,7 @@ public interface ClientProtocol {
/**
* Tell all datanodes to use a new, non-persistent bandwidth value for
- * dfs.balance.bandwidthPerSec.
+ * dfs.datanode.balance.bandwidthPerSec.
*
* @param bandwidth Blanacer bandwidth in bytes per second for this datanode.
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index e9cdfee..13d5846 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -114,7 +114,7 @@ import com.google.common.base.Preconditions;
* defined in the default configuration file:
* <pre>
* <property>
- * <name>dfs.balance.bandwidthPerSec</name>
+ * <name>dfs.datanode.balance.bandwidthPerSec</name>
* <value>1048576</value>
* <description> Specifies the maximum bandwidth that each datanode
* can utilize for the balancing purpose in term of the number of bytes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 8ab4bba..c180432 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -184,7 +184,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
private boolean forceRegistration = false;
// A system administrator can tune the balancer bandwidth parameter
- // (dfs.balance.bandwidthPerSec) dynamically by calling
+ // (dfs.datanode.balance.bandwidthPerSec) dynamically by calling
// "dfsadmin -setBalanacerBandwidth <newbandwidth>", at which point the
// following 'bandwidth' variable gets updated with the new value for each
// node. Once the heartbeat command is issued to update the value on the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index fed1864..11eb6ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1738,7 +1738,7 @@ public class DatanodeManager {
/**
* Tell all datanodes to use a new, non-persistent bandwidth value for
- * dfs.balance.bandwidthPerSec.
+ * dfs.datanode.balance.bandwidthPerSec.
*
* A system administrator can tune the balancer bandwidth parameter
* (dfs.datanode.balance.bandwidthPerSec) dynamically by calling
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java
index 7f17800..49bc329 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.protocol;
/*
* A system administrator can tune the balancer bandwidth parameter
- * (dfs.balance.bandwidthPerSec) dynamically by calling
+ * (dfs.datanode.balance.bandwidthPerSec) dynamically by calling
* "dfsadmin -setBalanacerBandwidth newbandwidth".
* This class is to define the command which sends the new bandwidth value to
* each datanode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index a60f24b..4dd9d7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -1078,7 +1078,7 @@ public class DFSAdmin extends FsShell {
"\tHDFS block balancing.\n\n" +
"\t\t<bandwidth> is the maximum number of bytes per second\n" +
"\t\tthat will be used by each datanode. This value overrides\n" +
- "\t\tthe dfs.balance.bandwidthPerSec parameter.\n\n" +
+ "\t\tthe dfs.datanode.balance.bandwidthPerSec parameter.\n\n" +
"\t\t--- NOTE: The new value is not persistent on the DataNode.---\n";
String getBalancerBandwidth = "-getBalancerBandwidth <datanode_host:ipc_port>:\n" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4164a203/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 46cb839..1a4465c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -394,7 +394,7 @@ Usage:
| `-printTopology` | Print a tree of the racks and their nodes as reported by the Namenode |
| `-refreshNamenodes` datanodehost:port | For the given datanode, reloads the configuration files, stops serving the removed block-pools and starts serving new block-pools. |
| `-deleteBlockPool` datanode-host:port blockpoolId [force] | If force is passed, block pool directory for the given blockpool id on the given datanode is deleted along with its contents, otherwise the directory is deleted only if it is empty. The command will fail if datanode is still serving the block pool. Refer to refreshNamenodes to shutdown a block pool service on a datanode. |
-| `-setBalancerBandwidth` \<bandwidth in bytes per second\> | Changes the network bandwidth used by each datanode during HDFS block balancing. \<bandwidth\> is the maximum number of bytes per second that will be used by each datanode. This value overrides the dfs.balance.bandwidthPerSec parameter. NOTE: The new value is not persistent on the DataNode. |
+| `-setBalancerBandwidth` \<bandwidth in bytes per second\> | Changes the network bandwidth used by each datanode during HDFS block balancing. \<bandwidth\> is the maximum number of bytes per second that will be used by each datanode. This value overrides the dfs.datanode.balance.bandwidthPerSec parameter. NOTE: The new value is not persistent on the DataNode. |
| `-getBalancerBandwidth` \<datanode\_host:ipc\_port\> | Get the network bandwidth(in bytes per second) for the given datanode. This is the maximum network bandwidth used by the datanode during HDFS block balancing.|
| `-fetchImage` \<local directory\> | Downloads the most recent fsimage from the NameNode and saves it in the specified local directory. |
| `-allowSnapshot` \<snapshotDir\> | Allowing snapshots of a directory to be created. If the operation completes successfully, the directory becomes snapshottable. See the [HDFS Snapshot Documentation](./HdfsSnapshots.html) for more information. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-6061. Add an
UncaughtExceptionHandler for critical threads in RM. (Yufei Gu via kasha)
Posted by su...@apache.org.
YARN-6061. Add an UncaughtExceptionHandler for critical threads in RM. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/652679aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/652679aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/652679aa
Branch: refs/heads/YARN-2915
Commit: 652679aa8ad6f9e61b8ed8e2b04b3e0332025e94
Parents: aaf2713
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Tue Feb 14 13:39:34 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Tue Feb 14 13:39:41 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/client/TestRMFailover.java | 100 ++++++++++++++++++-
.../yarn/server/resourcemanager/RMContext.java | 2 +
.../server/resourcemanager/RMContextImpl.java | 10 ++
...MCriticalThreadUncaughtExceptionHandler.java | 58 +++++++++++
.../resourcemanager/RMFatalEventType.java | 5 +-
.../server/resourcemanager/ResourceManager.java | 65 +++++++++---
.../resourcemanager/recovery/RMStateStore.java | 13 +--
.../DominantResourceFairnessPolicy.java | 2 +-
8 files changed, 226 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index b58a775..4bf6a78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -22,7 +22,10 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.net.HttpURLConnection;
@@ -37,14 +40,18 @@ import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMCriticalThreadUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.junit.After;
@@ -174,7 +181,7 @@ public class TestRMFailover extends ClientBaseWithFixes {
// so it transitions to standby.
ResourceManager rm = cluster.getResourceManager(
cluster.getActiveRMIndex());
- rm.handleTransitionToStandBy();
+ rm.handleTransitionToStandByInNewThread();
int maxWaitingAttempts = 2000;
while (maxWaitingAttempts-- > 0 ) {
if (rm.getRMContext().getHAServiceState() == HAServiceState.STANDBY) {
@@ -349,4 +356,95 @@ public class TestRMFailover extends ClientBaseWithFixes {
}
return redirectUrl;
}
+
+ /**
+ * Throw {@link RuntimeException} inside a thread of
+ * {@link ResourceManager} with HA enabled and check if the
+ * {@link ResourceManager} is transited to standby state.
+ *
+ * @throws InterruptedException if any
+ */
+ @Test
+ public void testUncaughtExceptionHandlerWithHAEnabled()
+ throws InterruptedException {
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
+ cluster.init(conf);
+ cluster.start();
+ assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+
+ ResourceManager resourceManager = cluster.getResourceManager(
+ cluster.getActiveRMIndex());
+
+ final RMCriticalThreadUncaughtExceptionHandler exHandler =
+ new RMCriticalThreadUncaughtExceptionHandler(
+ resourceManager.getRMContext());
+
+ // Create a thread and throw a RTE inside it
+ final RuntimeException rte = new RuntimeException("TestRuntimeException");
+ final Thread testThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ throw rte;
+ }
+ });
+ testThread.setName("TestThread");
+ testThread.setUncaughtExceptionHandler(exHandler);
+ testThread.start();
+ testThread.join();
+
+ int maxWaitingAttempts = 2000;
+ while (maxWaitingAttempts-- > 0) {
+ if (resourceManager.getRMContext().getHAServiceState()
+ == HAServiceState.STANDBY) {
+ break;
+ }
+ Thread.sleep(1);
+ }
+ assertFalse("RM didn't transition to Standby ", maxWaitingAttempts < 0);
+ }
+
+ /**
+ * Throw {@link RuntimeException} inside a thread of
+ * {@link ResourceManager} with HA disabled and check
+ * {@link RMCriticalThreadUncaughtExceptionHandler} instance.
+ *
+ * Used {@link ExitUtil} class to avoid jvm exit through
+ * {@code System.exit(-1)}.
+ *
+ * @throws InterruptedException if any
+ */
+ @Test
+ public void testUncaughtExceptionHandlerWithoutHA()
+ throws InterruptedException {
+ ExitUtil.disableSystemExit();
+
+ // Create a MockRM and start it
+ ResourceManager resourceManager = new MockRM();
+ ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
+ resourceManager.getRMContext().getStateStore().start();
+ resourceManager.getRMContext().getContainerTokenSecretManager().
+ rollMasterKey();
+
+ final RMCriticalThreadUncaughtExceptionHandler exHandler =
+ new RMCriticalThreadUncaughtExceptionHandler(
+ resourceManager.getRMContext());
+ final RMCriticalThreadUncaughtExceptionHandler spyRTEHandler =
+ spy(exHandler);
+
+ // Create a thread and throw a RTE inside it
+ final RuntimeException rte = new RuntimeException("TestRuntimeException");
+ final Thread testThread = new Thread(new Runnable() {
+ @Override public void run() {
+ throw rte;
+ }
+ });
+ testThread.setName("TestThread");
+ testThread.setUncaughtExceptionHandler(spyRTEHandler);
+ assertSame(spyRTEHandler, testThread.getUncaughtExceptionHandler());
+ testThread.start();
+ testThread.join();
+
+ verify(spyRTEHandler).uncaughtException(testThread, rte);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 26ef5ac..ba6b915 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -156,4 +156,6 @@ public interface RMContext {
RMAppLifetimeMonitor getRMAppLifetimeMonitor();
String getHAZookeeperConnectionState();
+
+ ResourceManager getResourceManager();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index a452f95..fb160c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -82,6 +82,7 @@ public class RMContextImpl implements RMContext {
private final Object haServiceStateLock = new Object();
+ private ResourceManager resourceManager;
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -522,4 +523,13 @@ public class RMContextImpl implements RMContext {
return elector.getZookeeperConnectionState();
}
}
+
+ @Override
+ public ResourceManager getResourceManager() {
+ return resourceManager;
+ }
+
+ public void setResourceManager(ResourceManager rm) {
+ this.resourceManager = rm;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
new file mode 100644
index 0000000..c5c6087
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.conf.HAUtil;
+
+/**
+ * This class either shuts down {@link ResourceManager} or transitions the
+ * {@link ResourceManager} to standby state if a critical thread throws an
+ * uncaught exception. It is intended to be installed by calling
+ * {@code setUncaughtExceptionHandler(Thread.UncaughtExceptionHandler)}
+ * in the thread entry point or after creation of threads.
+ */
+@Private
+public class RMCriticalThreadUncaughtExceptionHandler
+ implements UncaughtExceptionHandler {
+ private static final Log LOG = LogFactory.getLog(
+ RMCriticalThreadUncaughtExceptionHandler.class);
+ private RMContext rmContext;
+
+ public RMCriticalThreadUncaughtExceptionHandler(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.fatal("Critical thread " + t.getName() + " crashed!", e);
+
+ if (HAUtil.isHAEnabled(rmContext.getYarnConfiguration())) {
+ rmContext.getResourceManager().handleTransitionToStandByInNewThread();
+ } else {
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMFatalEvent(RMFatalEventType.CRITICAL_THREAD_CRASH,
+ new Exception(e)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
index 87cc496..b6f6b3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
@@ -29,5 +29,8 @@ public enum RMFatalEventType {
EMBEDDED_ELECTOR_FAILED,
// Source <- Admin Service
- TRANSITION_TO_ACTIVE_FAILED
+ TRANSITION_TO_ACTIVE_FAILED,
+
+ // Source <- Critical Thread Crash
+ CRITICAL_THREAD_CRASH
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 886f4ab..58e4077 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -130,6 +130,7 @@ import java.security.PrivilegedExceptionAction;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* The ResourceManager is the main class that is a set of components.
@@ -205,7 +206,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
private Configuration conf;
private UserGroupInformation rmLoginUGI;
-
+
public ResourceManager() {
super("ResourceManager");
}
@@ -232,7 +233,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
this.rmContext = new RMContextImpl();
-
+ rmContext.setResourceManager(this);
+
this.configurationProvider =
ConfigurationProviderFactory.getConfigurationProvider(conf);
this.configurationProvider.init(this.conf);
@@ -564,6 +566,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
private ResourceManager rm;
private RMActiveServiceContext activeServiceContext;
private boolean fromActive = false;
+ private StandByTransitionRunnable standByTransitionRunnable;
RMActiveServices(ResourceManager rm) {
super("RMActiveServices");
@@ -572,6 +575,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
@Override
protected void serviceInit(Configuration configuration) throws Exception {
+ standByTransitionRunnable = new StandByTransitionRunnable();
+
activeServiceContext = new RMActiveServiceContext();
rmContext.setActiveServiceContext(activeServiceContext);
@@ -819,19 +824,51 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
- public void handleTransitionToStandBy() {
- if (rmContext.isHAEnabled()) {
- try {
- // Transition to standby and reinit active services
- LOG.info("Transitioning RM to Standby mode");
- transitionToStandby(true);
- EmbeddedElector elector = rmContext.getLeaderElectorService();
- if (elector != null) {
- elector.rejoinElection();
+ /**
+ * Transition to standby state in a new thread. The transition operation is
+ * asynchronous to avoid deadlock caused by cyclic dependency.
+ */
+ public void handleTransitionToStandByInNewThread() {
+ Thread standByTransitionThread =
+ new Thread(activeServices.standByTransitionRunnable);
+ standByTransitionThread.setName("StandByTransitionThread");
+ standByTransitionThread.start();
+ }
+
+ /**
+ * The class to transition RM to standby state. The same
+ * {@link StandByTransitionRunnable} object could be used in multiple threads,
+ * but runs only once. That's because RM can go back to active state after
+ * transition to standby state, the same runnable in the old context can't
+ * transition RM to standby state again. A new runnable is created every time
+ * RM transitions to active state.
+ */
+ private class StandByTransitionRunnable implements Runnable {
+ // The atomic variable to make sure multiple threads with the same runnable
+ // run only once.
+ private AtomicBoolean hasAlreadyRun = new AtomicBoolean(false);
+
+ @Override
+ public void run() {
+ // Run this only once, even if multiple threads end up triggering
+ // this simultaneously.
+ if (hasAlreadyRun.getAndSet(true)) {
+ return;
+ }
+
+ if (rmContext.isHAEnabled()) {
+ try {
+ // Transition to standby and reinit active services
+ LOG.info("Transitioning RM to Standby mode");
+ transitionToStandby(true);
+ EmbeddedElector elector = rmContext.getLeaderElectorService();
+ if (elector != null) {
+ elector.rejoinElection();
+ }
+ } catch (Exception e) {
+ LOG.fatal("Failed to transition RM to Standby mode.", e);
+ ExitUtil.terminate(1, e);
}
- } catch (Exception e) {
- LOG.fatal("Failed to transition RM to Standby mode.", e);
- ExitUtil.terminate(1, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 6ede3b4..5e3cf22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -1132,10 +1132,7 @@ public abstract class RMStateStore extends AbstractService {
if (HAUtil.isHAEnabled(getConfig())) {
LOG.warn("State-store fenced ! Transitioning RM to standby");
isFenced = true;
- Thread standByTransitionThread =
- new Thread(new StandByTransitionThread());
- standByTransitionThread.setName("StandByTransitionThread Handler");
- standByTransitionThread.start();
+ resourceManager.handleTransitionToStandByInNewThread();
} else if (YarnConfiguration.shouldRMFailFast(getConfig())) {
LOG.fatal("Fail RM now due to state-store error!");
rmDispatcher.getEventHandler().handle(
@@ -1200,14 +1197,6 @@ public abstract class RMStateStore extends AbstractService {
this.resourceManager = rm;
}
- private class StandByTransitionThread implements Runnable {
- @Override
- public void run() {
- LOG.info("RMStateStore has been fenced");
- resourceManager.handleTransitionToStandBy();
- }
- }
-
public RMStateStoreState getRMStateStoreState() {
this.readLock.lock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652679aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index ad41b11..7a29735 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -90,7 +90,7 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
- return !Resources.fitsIn(usage, fairShare);
+ return Resources.greaterThan(CALCULATOR, null, usage, fairShare);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-5519. Add SubClusterId in
AddApplicationHomeSubClusterResponse for Router Failover. (Ellen Hui via
Subru)
Posted by su...@apache.org.
YARN-5519. Add SubClusterId in AddApplicationHomeSubClusterResponse for Router Failover. (Ellen Hui via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b0386e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b0386e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b0386e3
Branch: refs/heads/YARN-2915
Commit: 0b0386e3c156e345951c2e9deefc9eb459d687f0
Parents: ed2c6c7
Author: Subru Krishnan <su...@apache.org>
Authored: Mon Aug 15 14:47:02 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
...ederationApplicationHomeSubClusterStore.java | 21 +++---
.../store/impl/MemoryFederationStateStore.java | 22 +++---
.../AddApplicationHomeSubClusterResponse.java | 29 ++++++--
...ApplicationHomeSubClusterResponsePBImpl.java | 39 +++++++++++
.../proto/yarn_server_federation_protos.proto | 1 +
.../impl/FederationStateStoreBaseTest.java | 71 +++++++++-----------
6 files changed, 120 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b0386e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
index 22bb88a..ace2457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -51,15 +51,20 @@ public interface FederationApplicationHomeSubClusterStore {
/**
* Register the home {@code SubClusterId} of the newly submitted
* {@code ApplicationId}. Currently response is empty if the operation was
- * successful, if not an exception reporting reason for a failure.
+ * successful, if not an exception reporting reason for a failure. If a
+ * mapping for the application already existed, the {@code SubClusterId} in
+ * this response will return the existing mapping which might be different
+ * from that in the {@code AddApplicationHomeSubClusterRequest}.
*
* @param request the request to register a new application with its home
* sub-cluster
- * @return empty on successful registration of the application in the
- * StateStore, if not an exception reporting reason for a failure
+ * @return upon successful registration of the application in the StateStore,
+ * {@code AddApplicationHomeSubClusterRequest} containing the home
+ * sub-cluster of the application. Otherwise, an exception reporting
+ * reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -73,7 +78,7 @@ public interface FederationApplicationHomeSubClusterStore {
* not an exception reporting reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -85,7 +90,7 @@ public interface FederationApplicationHomeSubClusterStore {
* subcluster
* @throws YarnException if the request is invalid/fails
*/
- GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -96,7 +101,7 @@ public interface FederationApplicationHomeSubClusterStore {
* @return the mapping of all submitted application to it's home sub-cluster
* @throws YarnException if the request is invalid/fails
*/
- GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException;
/**
@@ -110,7 +115,7 @@ public interface FederationApplicationHomeSubClusterStore {
* not an exception reporting reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
DeleteApplicationHomeSubClusterRequest request) throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b0386e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index a540dff..8144435 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -89,7 +89,6 @@ public class MemoryFederationStateStore implements FederationStateStore {
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
- subClusterInfo.setLastStartTime(clock.getTime());
membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
return SubClusterRegisterResponse.newInstance();
}
@@ -156,21 +155,22 @@ public class MemoryFederationStateStore implements FederationStateStore {
// FederationApplicationHomeSubClusterStore methods
@Override
- public AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
- if (applications.containsKey(appId)) {
- throw new YarnException("Application " + appId + " already exists");
+
+ if (!applications.containsKey(appId)) {
+ applications.put(appId,
+ request.getApplicationHomeSubCluster().getHomeSubCluster());
}
- applications.put(appId,
- request.getApplicationHomeSubCluster().getHomeSubCluster());
- return AddApplicationHomeSubClusterResponse.newInstance();
+ return AddApplicationHomeSubClusterResponse
+ .newInstance(applications.get(appId));
}
@Override
- public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
@@ -184,7 +184,7 @@ public class MemoryFederationStateStore implements FederationStateStore {
}
@Override
- public GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
@@ -196,7 +196,7 @@ public class MemoryFederationStateStore implements FederationStateStore {
}
@Override
- public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
List<ApplicationHomeSubCluster> result =
new ArrayList<ApplicationHomeSubCluster>();
@@ -210,7 +210,7 @@ public class MemoryFederationStateStore implements FederationStateStore {
}
@Override
- public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
DeleteApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b0386e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
index 2145dd1..913f8e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
@@ -24,10 +24,13 @@ import org.apache.hadoop.yarn.util.Records;
/**
* AddApplicationHomeSubClusterResponse contains the answer from the
* {@code FederationApplicationHomeSubClusterStore} to a request to insert a
- * newly generated applicationId and its owner. Currently response is empty if
- * the operation was successful, if not an exception reporting reason for a
- * failure.
+ * newly generated applicationId and its owner.
*
+ * The response contains application's home sub-cluster as it is stored in the
+ * {@code FederationApplicationHomeSubClusterStore}. If a mapping for the
+ * application already existed, the {@code SubClusterId} in this response will
+ * return the existing mapping which might be different from that in the
+ * {@code AddApplicationHomeSubClusterRequest}.
*/
@Private
@Unstable
@@ -35,10 +38,28 @@ public abstract class AddApplicationHomeSubClusterResponse {
@Private
@Unstable
- public static AddApplicationHomeSubClusterResponse newInstance() {
+ public static AddApplicationHomeSubClusterResponse newInstance(
+ SubClusterId homeSubCluster) {
AddApplicationHomeSubClusterResponse response =
Records.newRecord(AddApplicationHomeSubClusterResponse.class);
+ response.setHomeSubCluster(homeSubCluster);
return response;
}
+ /**
+ * Set the home sub-cluster that this application has been assigned to.
+ *
+ * @param homeSubCluster the {@link SubClusterId} of this application's home
+ * sub-cluster
+ */
+ public abstract void setHomeSubCluster(SubClusterId homeSubCluster);
+
+ /**
+ * Get the home sub-cluster that this application has been assigned to. This
+ * may not match the {@link SubClusterId} in the corresponding response, if
+ * the mapping for the request's application already existed.
+ *
+ * @return the {@link SubClusterId} of this application's home sub-cluster
+ */
+ public abstract SubClusterId getHomeSubCluster();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b0386e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
index 1415981..b30c41c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import com.google.protobuf.TextFormat;
@@ -48,6 +51,13 @@ public class AddApplicationHomeSubClusterResponsePBImpl
viaProto = true;
}
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = AddApplicationHomeSubClusterResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
public AddApplicationHomeSubClusterResponseProto getProto() {
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -55,6 +65,27 @@ public class AddApplicationHomeSubClusterResponsePBImpl
}
@Override
+ public void setHomeSubCluster(SubClusterId homeSubCluster) {
+ maybeInitBuilder();
+ if (homeSubCluster == null) {
+ builder.clearHomeSubCluster();
+ return;
+ }
+ builder.setHomeSubCluster(convertToProtoFormat(homeSubCluster));
+ }
+
+ @Override
+ public SubClusterId getHomeSubCluster() {
+ AddApplicationHomeSubClusterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+
+ if (!p.hasHomeSubCluster()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getHomeSubCluster());
+ }
+
+ @Override
public int hashCode() {
return getProto().hashCode();
}
@@ -75,4 +106,12 @@ public class AddApplicationHomeSubClusterResponsePBImpl
return TextFormat.shortDebugString(getProto());
}
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+ return new SubClusterIdPBImpl(sc);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+ return ((SubClusterIdPBImpl) sc).getProto();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b0386e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 11f786f..cedf482 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -102,6 +102,7 @@ message AddApplicationHomeSubClusterRequestProto {
}
message AddApplicationHomeSubClusterResponseProto {
+ optional SubClusterIdProto home_sub_cluster = 1;
}
message UpdateApplicationHomeSubClusterRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b0386e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 165dd78..414696b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -94,7 +94,7 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testDeregisterSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- registerSubCluster(subClusterId);
+ registerSubCluster(createSubClusterInfo(subClusterId));
SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
.newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
@@ -124,7 +124,7 @@ public abstract class FederationStateStoreBaseTest {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
- registerSubCluster(subClusterId);
+ registerSubCluster(subClusterInfo);
GetSubClusterInfoRequest request =
GetSubClusterInfoRequest.newInstance(subClusterId);
@@ -184,10 +184,10 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testSubClusterHeartbeat() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- registerSubCluster(subClusterId);
+ registerSubCluster(createSubClusterInfo(subClusterId));
SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
- .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "capability");
stateStore.subClusterHeartbeat(heartbeatRequest);
Assert.assertEquals(SubClusterState.SC_RUNNING,
@@ -199,7 +199,7 @@ public abstract class FederationStateStoreBaseTest {
public void testSubClusterHeartbeatUnknownSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
- .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "capability");
try {
stateStore.subClusterHeartbeat(heartbeatRequest);
@@ -213,7 +213,7 @@ public abstract class FederationStateStoreBaseTest {
// Test FederationApplicationHomeSubClusterStore
@Test
- public void testAddApplicationHomeSubClusterMap() throws Exception {
+ public void testAddApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId = SubClusterId.newInstance("SC");
ApplicationHomeSubCluster ahsc =
@@ -222,15 +222,15 @@ public abstract class FederationStateStoreBaseTest {
AddApplicationHomeSubClusterRequest request =
AddApplicationHomeSubClusterRequest.newInstance(ahsc);
AddApplicationHomeSubClusterResponse response =
- stateStore.addApplicationHomeSubClusterMap(request);
+ stateStore.addApplicationHomeSubCluster(request);
- Assert.assertNotNull(response);
+ Assert.assertEquals(subClusterId, response.getHomeSubCluster());
Assert.assertEquals(subClusterId, queryApplicationHomeSC(appId));
}
@Test
- public void testAddApplicationHomeSubClusterMapAppAlreadyExists()
+ public void testAddApplicationHomeSubClusterAppAlreadyExists()
throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
@@ -240,21 +240,17 @@ public abstract class FederationStateStoreBaseTest {
ApplicationHomeSubCluster ahsc2 =
ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
- try {
- stateStore.addApplicationHomeSubClusterMap(
- AddApplicationHomeSubClusterRequest.newInstance(ahsc2));
- Assert.fail();
- } catch (YarnException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith("Application " + appId.toString() + " already exists"));
- }
+ AddApplicationHomeSubClusterResponse response =
+ stateStore.addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc2));
+ Assert.assertEquals(subClusterId1, response.getHomeSubCluster());
Assert.assertEquals(subClusterId1, queryApplicationHomeSC(appId));
}
@Test
- public void testDeleteApplicationHomeSubClusterMap() throws Exception {
+ public void testDeleteApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId = SubClusterId.newInstance("SC");
addApplicationHomeSC(appId, subClusterId);
@@ -263,7 +259,7 @@ public abstract class FederationStateStoreBaseTest {
DeleteApplicationHomeSubClusterRequest.newInstance(appId);
DeleteApplicationHomeSubClusterResponse response =
- stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+ stateStore.deleteApplicationHomeSubCluster(delRequest);
Assert.assertNotNull(response);
try {
@@ -277,14 +273,13 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testDeleteApplicationHomeSubClusterMapUnknownApp()
- throws Exception {
+ public void testDeleteApplicationHomeSubClusterUnknownApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
DeleteApplicationHomeSubClusterRequest delRequest =
DeleteApplicationHomeSubClusterRequest.newInstance(appId);
try {
- stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+ stateStore.deleteApplicationHomeSubCluster(delRequest);
Assert.fail();
} catch (YarnException e) {
Assert.assertTrue(e.getMessage()
@@ -293,7 +288,7 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testGetApplicationHomeSubClusterMap() throws Exception {
+ public void testGetApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId = SubClusterId.newInstance("SC");
addApplicationHomeSC(appId, subClusterId);
@@ -302,7 +297,7 @@ public abstract class FederationStateStoreBaseTest {
GetApplicationHomeSubClusterRequest.newInstance(appId);
GetApplicationHomeSubClusterResponse result =
- stateStore.getApplicationHomeSubClusterMap(getRequest);
+ stateStore.getApplicationHomeSubCluster(getRequest);
Assert.assertEquals(appId,
result.getApplicationHomeSubCluster().getApplicationId());
@@ -311,13 +306,13 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testGetApplicationHomeSubClusterMapUnknownApp() throws Exception {
+ public void testGetApplicationHomeSubClusterUnknownApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
GetApplicationHomeSubClusterRequest request =
GetApplicationHomeSubClusterRequest.newInstance(appId);
try {
- stateStore.getApplicationHomeSubClusterMap(request);
+ stateStore.getApplicationHomeSubCluster(request);
Assert.fail();
} catch (YarnException e) {
Assert.assertTrue(e.getMessage()
@@ -326,7 +321,7 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testGetApplicationsHomeSubClusterMap() throws Exception {
+ public void testGetApplicationsHomeSubCluster() throws Exception {
ApplicationId appId1 = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
ApplicationHomeSubCluster ahsc1 =
@@ -344,7 +339,7 @@ public abstract class FederationStateStoreBaseTest {
GetApplicationsHomeSubClusterRequest.newInstance();
GetApplicationsHomeSubClusterResponse result =
- stateStore.getApplicationsHomeSubClusterMap(getRequest);
+ stateStore.getApplicationsHomeSubCluster(getRequest);
Assert.assertEquals(2, result.getAppsHomeSubClusters().size());
Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc1));
@@ -352,7 +347,7 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testUpdateApplicationHomeSubClusterMap() throws Exception {
+ public void testUpdateApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
addApplicationHomeSC(appId, subClusterId1);
@@ -365,16 +360,14 @@ public abstract class FederationStateStoreBaseTest {
UpdateApplicationHomeSubClusterRequest.newInstance(ahscUpdate);
UpdateApplicationHomeSubClusterResponse response =
- stateStore.updateApplicationHomeSubClusterMap(updateRequest);
+ stateStore.updateApplicationHomeSubCluster(updateRequest);
Assert.assertNotNull(response);
-
Assert.assertEquals(subClusterId2, queryApplicationHomeSC(appId));
}
@Test
- public void testUpdateApplicationHomeSubClusterMapUnknownApp()
- throws Exception {
+ public void testUpdateApplicationHomeSubClusterUnknownApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
ApplicationHomeSubCluster ahsc =
@@ -384,7 +377,7 @@ public abstract class FederationStateStoreBaseTest {
UpdateApplicationHomeSubClusterRequest.newInstance(ahsc);
try {
- stateStore.updateApplicationHomeSubClusterMap((updateRequest));
+ stateStore.updateApplicationHomeSubCluster((updateRequest));
Assert.fail();
} catch (YarnException e) {
Assert.assertTrue(e.getMessage()
@@ -484,7 +477,7 @@ public abstract class FederationStateStoreBaseTest {
return SubClusterInfo.newInstance(subClusterId, amRMAddress,
clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
- CLOCK.getTime(), "cabability");
+ CLOCK.getTime(), "capability");
}
private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
@@ -499,7 +492,7 @@ public abstract class FederationStateStoreBaseTest {
ApplicationHomeSubCluster.newInstance(appId, subClusterId);
AddApplicationHomeSubClusterRequest request =
AddApplicationHomeSubClusterRequest.newInstance(ahsc);
- stateStore.addApplicationHomeSubClusterMap(request);
+ stateStore.addApplicationHomeSubCluster(request);
}
private void setPolicyConf(String queue, String policyType)
@@ -510,10 +503,8 @@ public abstract class FederationStateStoreBaseTest {
stateStore.setPolicyConfiguration(request);
}
- private void registerSubCluster(SubClusterId subClusterId)
+ private void registerSubCluster(SubClusterInfo subClusterInfo)
throws YarnException {
-
- SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
stateStore.registerSubCluster(
SubClusterRegisterRequest.newInstance(subClusterInfo));
}
@@ -531,7 +522,7 @@ public abstract class FederationStateStoreBaseTest {
GetApplicationHomeSubClusterRequest.newInstance(appId);
GetApplicationHomeSubClusterResponse response =
- stateStore.getApplicationHomeSubClusterMap(request);
+ stateStore.getApplicationHomeSubCluster(request);
return response.getApplicationHomeSubCluster().getHomeSubCluster();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-5676. Add a
HashBasedRouterPolicy,
and small policies and test refactoring. (Carlo Curino via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
index 4975a9f..5fa02d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.PriorityBroadcastPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
deleted file mode 100644
index 542a5ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
-import org.junit.Before;
-
-/**
- * Simple test of {@link UniformBroadcastPolicyManager}.
- */
-public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
-
- @Before
- public void setup() {
- //config policy
- wfp = new UniformBroadcastPolicyManager();
- wfp.setQueue("queue1");
-
- //set expected params that the base test class will use for tests
- expectedPolicyManager = UniformBroadcastPolicyManager.class;
- expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
- expectedRouterPolicy = UniformRandomRouterPolicy.class;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
deleted file mode 100644
index ab9cec4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Simple test of {@link WeightedLocalityPolicyManager}.
- */
-public class TestWeightedLocalityPolicyManager extends
- BasePolicyManagerTest {
-
- private WeightedPolicyInfo policyInfo;
-
- @Before
- public void setup() {
- // configure a policy
-
- wfp = new WeightedLocalityPolicyManager();
- wfp.setQueue("queue1");
- SubClusterId sc1 = SubClusterId.newInstance("sc1");
- SubClusterId sc2 = SubClusterId.newInstance("sc2");
- policyInfo = new WeightedPolicyInfo();
-
- Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
- routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
- routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
- policyInfo.setRouterPolicyWeights(routerWeights);
-
- Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
- amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f);
- amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f);
- policyInfo.setAMRMPolicyWeights(amrmWeights);
-
- ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo(
- policyInfo);
-
- //set expected params that the base test class will use for tests
- expectedPolicyManager = WeightedLocalityPolicyManager.class;
- expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class;
- expectedRouterPolicy = WeightedRandomRouterPolicy.class;
- }
-
- @Test
- public void testPolicyInfoSetCorrectly() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy,
- expectedRouterPolicy);
-
- //check the policyInfo propagates through ser/der correctly
- Assert.assertEquals(((WeightedLocalityPolicyManager) wfp)
- .getWeightedPolicyInfo(), policyInfo);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java
new file mode 100644
index 0000000..3cf73b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class provides common test methods for testing {@code
+ * FederationPolicyManager}s.
+ */
+public abstract class BasePolicyManagerTest {
+
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected FederationPolicyManager wfp = null;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedPolicyManager;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedAMRMProxyPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedRouterPolicy;
+
+ @Test
+ public void testSerializeAndInstantiate() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testSerializeAndInstantiateBad1() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, String.class,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad2() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ String.class, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad3() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, String.class);
+ }
+
+ protected static void serializeAndDeserializePolicyManager(
+ FederationPolicyManager wfp, Class policyManagerType,
+ Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception {
+
+ // serializeConf it in a context
+ SubClusterPolicyConfiguration fpc = wfp.serializeConf();
+ fpc.setType(policyManagerType.getCanonicalName());
+ FederationPolicyInitializationContext context =
+ new FederationPolicyInitializationContext();
+ context.setSubClusterPolicyConfiguration(fpc);
+ context
+ .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ context.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster"));
+
+ // based on the "context" created instantiate new class and use it
+ Class c = Class.forName(wfp.getClass().getCanonicalName());
+ FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance();
+
+ FederationAMRMProxyPolicy federationAMRMProxyPolicy =
+ wfp2.getAMRMPolicy(context, null);
+
+ // needed only for tests (getARMRMPolicy change the "type" in conf)
+ fpc.setType(wfp.getClass().getCanonicalName());
+
+ FederationRouterPolicy federationRouterPolicy =
+ wfp2.getRouterPolicy(context, null);
+
+ Assert.assertEquals(federationAMRMProxyPolicy.getClass(),
+ expAMRMProxyPolicy);
+
+ Assert.assertEquals(federationRouterPolicy.getClass(), expRouterPolicy);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java
new file mode 100644
index 0000000..5fc4a56
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.HashBasedRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link HashBroadcastPolicyManager}.
+ */
+public class TestHashBasedBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ // config policy
+ wfp = new HashBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = HashBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = HashBasedRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..21b39e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test of {@link PriorityBroadcastPolicyManager}.
+ */
+public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new PriorityBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo);
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = PriorityBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = PriorityRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+
+ // check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(
+ ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(),
+ policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..57fafdc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link UniformBroadcastPolicyManager}.
+ */
+public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ //config policy
+ wfp = new UniformBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = UniformBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = UniformRandomRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..51661473
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple test of {@link WeightedLocalityPolicyManager}.
+ */
+public class TestWeightedLocalityPolicyManager extends
+ BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new WeightedLocalityPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+ amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setAMRMPolicyWeights(amrmWeights);
+
+ ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo(
+ policyInfo);
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = WeightedLocalityPolicyManager.class;
+ expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ expectedRouterPolicy = WeightedRandomRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy,
+ expectedRouterPolicy);
+
+ //check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(((WeightedLocalityPolicyManager) wfp)
+ .getWeightedPolicyInfo(), policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
new file mode 100644
index 0000000..2e7a0af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Base class for router policies tests, tests for null input cases.
+ */
+public abstract class BaseRouterPoliciesTest
+ extends BaseFederationPoliciesTest {
+
+ @Test
+ public void testNullQueueRouting() throws YarnException {
+ FederationRouterPolicy localPolicy = (FederationRouterPolicy) getPolicy();
+ ApplicationSubmissionContext applicationSubmissionContext =
+ ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+ false, false, 0, Resources.none(), null, false, null, null);
+ SubClusterId chosen =
+ localPolicy.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertNotNull(chosen);
+ }
+
+ @Test(expected = FederationPolicyException.class)
+ public void testNullAppContext() throws YarnException {
+ ((FederationRouterPolicy) getPolicy()).getHomeSubcluster(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
new file mode 100644
index 0000000..af7fe43
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link HashBasedRouterPolicy}. Tests that one of
+ * the active sub-cluster is chosen.
+ */
+public class TestHashBasedRouterPolicy extends BaseRouterPoliciesTest {
+
+ private int numSubclusters = 10;
+
+ @Before
+ public void setUp() throws Exception {
+
+ // set policy in base class
+ setPolicy(new HashBasedRouterPolicy());
+
+ // setting up the active sub-clusters for this test
+ setMockActiveSubclusters(numSubclusters);
+
+ // initialize policy with context
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+ }
+
+ @Test
+ public void testHashSpreadUniformlyAmongSubclusters() throws YarnException {
+ SubClusterId chosen;
+
+ Map<SubClusterId, AtomicLong> counter = new HashMap<>();
+ for (SubClusterId id : getActiveSubclusters().keySet()) {
+ counter.put(id, new AtomicLong(0));
+ }
+
+ long jobPerSub = 100;
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ for (int i = 0; i < jobPerSub * numSubclusters; i++) {
+ when(applicationSubmissionContext.getQueue()).thenReturn("queue" + i);
+ chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(applicationSubmissionContext);
+ counter.get(chosen).addAndGet(1);
+ }
+
+ // hash spread the jobs equally among the subclusters
+ for (AtomicLong a : counter.values()) {
+ Assert.assertEquals(a.get(), jobPerSub);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index 906e35f..b70b4aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -36,7 +35,7 @@ import org.junit.Test;
* Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load
* is properly considered for allocation.
*/
-public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
+public class TestLoadBasedRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
index eefcfd9..42d919d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -23,7 +23,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -38,7 +37,7 @@ import org.junit.Test;
* Simple test class for the {@link PriorityRouterPolicy}. Tests that the
* weights are correctly used for ordering the choice of sub-clusters.
*/
-public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest {
+public class TestPriorityRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
index ac41ab5..b45aa2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
@@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -36,7 +35,7 @@ import org.junit.Test;
* Simple test class for the {@link UniformRandomRouterPolicy}. Tests that one
* of the active subcluster is chosen.
*/
-public class TestUniformRandomRouterPolicy extends BaseFederationPoliciesTest {
+public class TestUniformRandomRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
index 78967d0..34cc5f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -24,8 +24,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -41,7 +41,7 @@ import org.junit.Test;
* number of randomized tests to check we are weighiting correctly even if
* clusters go inactive.
*/
-public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
+public class TestWeightedRandomRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
@@ -78,13 +78,18 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
@Test
public void testClusterChosenWithRightProbability() throws YarnException {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ when(context.getQueue()).thenReturn("queue1");
+ setApplicationSubmissionContext(context);
+
Map<SubClusterId, AtomicLong> counter = new HashMap<>();
for (SubClusterIdInfo id : getPolicyInfo().getRouterPolicyWeights()
.keySet()) {
counter.put(id.toId(), new AtomicLong(0));
}
- float numberOfDraws = 1000000;
+ float numberOfDraws = 100000;
for (float i = 0; i < numberOfDraws; i++) {
SubClusterId chosenId = ((FederationRouterPolicy) getPolicy())
@@ -113,8 +118,8 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
Assert.assertTrue(
"Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ " expected weight: " + expectedWeight,
- expectedWeight == 0 || (actualWeight / expectedWeight) < 1.1
- && (actualWeight / expectedWeight) > 0.9);
+ expectedWeight == 0 || (actualWeight / expectedWeight) < 1.2
+ && (actualWeight / expectedWeight) > 0.8);
} else {
Assert
.assertTrue(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-5601. Make the RM epoch base
value configurable. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-5601. Make the RM epoch base value configurable. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed985a3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed985a3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed985a3d
Branch: refs/heads/YARN-2915
Commit: ed985a3dd5e61aff57312447b6810163c3972ffe
Parents: 2994dff
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 2 12:23:57 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 5 ++++-
.../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++
.../apache/hadoop/yarn/conf/TestYarnConfigurationFields.java | 2 ++
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 7 +++++++
.../resourcemanager/recovery/FileSystemRMStateStore.java | 2 +-
.../server/resourcemanager/recovery/LeveldbRMStateStore.java | 2 +-
.../server/resourcemanager/recovery/MemoryRMStateStore.java | 1 +
.../yarn/server/resourcemanager/recovery/RMStateStore.java | 4 ++++
.../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 2 +-
.../resourcemanager/recovery/RMStateStoreTestBase.java | 8 +++++---
.../server/resourcemanager/recovery/TestFSRMStateStore.java | 1 +
.../resourcemanager/recovery/TestLeveldbRMStateStore.java | 1 +
.../server/resourcemanager/recovery/TestZKRMStateStore.java | 1 +
13 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 2f5451d..bbd03a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -293,7 +293,10 @@
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore" />
- <Field name="resourceManager"/>
+ <Or>
+ <Field name="resourceManager"/>
+ <Field name="baseEpoch"/>
+ </Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f3062e2..58b2f44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -142,6 +142,9 @@ public class YarnConfiguration extends Configuration {
public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
+ public static final String RM_EPOCH = RM_PREFIX + "epoch";
+ public static final long DEFAULT_RM_EPOCH = 0L;
+
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 5e0876f..3f3a06c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -75,6 +75,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.RM_EPOCH);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index e4bd64e..f84e5fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -781,6 +781,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
LOG.error("Failed to load/recover state", e);
throw e;
}
+ } else {
+ if (HAUtil.isFederationEnabled(conf)) {
+ long epoch = conf.getLong(YarnConfiguration.RM_EPOCH,
+ YarnConfiguration.DEFAULT_RM_EPOCH);
+ rmContext.setEpoch(epoch);
+ LOG.info("Epoch set for Federation: " + epoch);
+ }
}
super.serviceStart();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 929e2da..6468974 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -197,7 +197,7 @@ public class FileSystemRMStateStore extends RMStateStore {
@Override
public synchronized long getAndIncrementEpoch() throws Exception {
Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE);
- long currentEpoch = 0;
+ long currentEpoch = baseEpoch;
FileStatus status = getFileStatusWithRetries(epochNodePath);
if (status != null) {
// load current epoch
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index 02f90dd..dfe78e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -250,7 +250,7 @@ public class LeveldbRMStateStore extends RMStateStore {
@Override
public synchronized long getAndIncrementEpoch() throws Exception {
- long currentEpoch = 0;
+ long currentEpoch = baseEpoch;
byte[] dbKeyBytes = bytes(EPOCH_NODE);
try {
byte[] data = db.get(dbKeyBytes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
index 5f3328b..5041000 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
@@ -83,6 +83,7 @@ public class MemoryRMStateStore extends RMStateStore {
@Override
public synchronized void initInternal(Configuration conf) {
+ epoch = baseEpoch;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 5e3cf22..2a1948a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -99,6 +99,7 @@ public abstract class RMStateStore extends AbstractService {
"ReservationSystemRoot";
protected static final String VERSION_NODE = "RMVersionNode";
protected static final String EPOCH_NODE = "EpochNode";
+ protected long baseEpoch;
protected ResourceManager resourceManager;
private final ReadLock readLock;
private final WriteLock writeLock;
@@ -684,6 +685,9 @@ public abstract class RMStateStore extends AbstractService {
dispatcher.register(RMStateStoreEventType.class,
rmStateStoreEventHandler);
dispatcher.setDrainEventsOnStop();
+ // read the base epoch value from conf
+ baseEpoch = conf.getLong(YarnConfiguration.RM_EPOCH,
+ YarnConfiguration.DEFAULT_RM_EPOCH);
initInternal(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 1212a91..3cc7faa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -353,7 +353,7 @@ public class ZKRMStateStore extends RMStateStore {
@Override
public synchronized long getAndIncrementEpoch() throws Exception {
String epochNodePath = getNodePath(zkRootNodePath, EPOCH_NODE);
- long currentEpoch = 0;
+ long currentEpoch = baseEpoch;
if (exists(epochNodePath)) {
// load current epoch
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 514e9a0..771c16e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -91,6 +91,8 @@ public class RMStateStoreTestBase {
public static final Log LOG = LogFactory.getLog(RMStateStoreTestBase.class);
+ protected final long epoch = 10L;
+
static class TestDispatcher implements Dispatcher, EventHandler<Event> {
ApplicationAttemptId attemptId;
@@ -558,13 +560,13 @@ public class RMStateStoreTestBase {
store.setRMDispatcher(new TestDispatcher());
long firstTimeEpoch = store.getAndIncrementEpoch();
- Assert.assertEquals(0, firstTimeEpoch);
+ Assert.assertEquals(epoch, firstTimeEpoch);
long secondTimeEpoch = store.getAndIncrementEpoch();
- Assert.assertEquals(1, secondTimeEpoch);
+ Assert.assertEquals(epoch + 1, secondTimeEpoch);
long thirdTimeEpoch = store.getAndIncrementEpoch();
- Assert.assertEquals(2, thirdTimeEpoch);
+ Assert.assertEquals(epoch + 2, thirdTimeEpoch);
}
public void testAppDeletion(RMStateStoreHelper stateStoreHelper)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 6f0d53f..8c968cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -116,6 +116,7 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
900L);
+ conf.setLong(YarnConfiguration.RM_EPOCH, epoch);
if (adminCheckEnable) {
conf.setBoolean(
YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
index 4297e73..d6f77d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
@@ -81,6 +81,7 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase {
@Test(timeout = 60000)
public void testEpoch() throws Exception {
+ conf.setLong(YarnConfiguration.RM_EPOCH, epoch);
LeveldbStateStoreTester tester = new LeveldbStateStoreTester();
testEpoch(tester);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed985a3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index f71cf25..22eadea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -155,6 +155,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
conf.set(YarnConfiguration.RM_ZK_ADDRESS,
curatorTestingServer.getConnectString());
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
+ conf.setLong(YarnConfiguration.RM_EPOCH, epoch);
this.store = new TestZKRMStateStoreInternal(conf, workingZnode);
return this.store;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDFS-11391. Numeric usernames do
no work with WebHDFS FS write access. (Pierre Villard via Yongjun Zhang)
Posted by su...@apache.org.
HDFS-11391. Numeric usernames do no work with WebHDFS FS write access. (Pierre Villard via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e53f2b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e53f2b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e53f2b9
Branch: refs/heads/YARN-2915
Commit: 8e53f2b9b08560bf4f8e81e697063277dbdc68f9
Parents: 652679a
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Tue Feb 14 12:47:06 2017 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Tue Feb 14 13:40:53 2017 -0800
----------------------------------------------------------------------
.../hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e53f2b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index 095f41d..f8c15fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.JsonUtil;
@@ -55,6 +56,7 @@ import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -108,6 +110,10 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
throws IOException {
this.conf = conf;
this.confForCreate = confForCreate;
+ /** set user pattern based on configuration file */
+ UserParam.setUserPattern(
+ conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
+ DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-6170. TimelineReaderServer
should wait to join with HttpServer2 (Sangjin Lee via Varun Saxena)
Posted by su...@apache.org.
YARN-6170. TimelineReaderServer should wait to join with HttpServer2 (Sangjin Lee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/649deb72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/649deb72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/649deb72
Branch: refs/heads/YARN-2915
Commit: 649deb72fbb62568b4ea0d67444df6faaaed169d
Parents: 0aacd8f
Author: Varun Saxena <va...@apache.org>
Authored: Sat Feb 11 19:21:45 2017 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Sat Feb 11 19:21:45 2017 +0530
----------------------------------------------------------------------
.../timelineservice/reader/TimelineReaderServer.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/649deb72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index 76f7495..2835c1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -110,6 +110,16 @@ public class TimelineReaderServer extends CompositeService {
startTimelineReaderWebApp();
}
+ private void join() {
+ // keep the main thread that started the server up until it receives a stop
+ // signal
+ if (readerWebServer != null) {
+ try {
+ readerWebServer.join();
+ } catch (InterruptedException ignore) {}
+ }
+ }
+
@Override
protected void serviceStop() throws Exception {
if (readerWebServer != null) {
@@ -194,6 +204,7 @@ public class TimelineReaderServer extends CompositeService {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
- startTimelineReaderServer(args, conf);
+ TimelineReaderServer server = startTimelineReaderServer(args, conf);
+ server.join();
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-5325. Stateless ARMRMProxy
policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index e57709f..5de749f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -17,8 +17,8 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
@@ -30,34 +30,27 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
-import java.util.Map;
-
/**
* This implements a simple load-balancing policy. The policy "weights" are
* binary 0/1 values that enable/disable each sub-cluster, and the policy peaks
* the sub-cluster with the least load to forward this application.
*/
-public class LoadBasedRouterPolicy
- extends BaseWeightedRouterPolicy {
-
- private static final Log LOG =
- LogFactory.getLog(LoadBasedRouterPolicy.class);
+public class LoadBasedRouterPolicy extends AbstractRouterPolicy {
@Override
- public void reinitialize(FederationPolicyInitializationContext
- federationPolicyContext)
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
// remember old policyInfo
WeightedPolicyInfo tempPolicy = getPolicyInfo();
- //attempt new initialization
- super.reinitialize(federationPolicyContext);
+ // attempt new initialization
+ super.reinitialize(policyContext);
- //check extra constraints
+ // check extra constraints
for (Float weight : getPolicyInfo().getRouterPolicyWeights().values()) {
if (weight != 0 && weight != 1) {
- //reset to old policyInfo if check fails
+ // reset to old policyInfo if check fails
setPolicyInfo(tempPolicy);
throw new FederationPolicyInitializationException(
this.getClass().getCanonicalName()
@@ -69,18 +62,16 @@ public class LoadBasedRouterPolicy
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
- Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
- .getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> weights =
+ getPolicyInfo().getRouterPolicyWeights();
SubClusterIdInfo chosen = null;
long currBestMem = -1;
- for (Map.Entry<SubClusterId, SubClusterInfo> entry :
- activeSubclusters
+ for (Map.Entry<SubClusterId, SubClusterInfo> entry : activeSubclusters
.entrySet()) {
SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey());
if (weights.containsKey(id) && weights.get(id) > 0) {
@@ -95,8 +86,7 @@ public class LoadBasedRouterPolicy
return chosen.toId();
}
- private long getAvailableMemory(SubClusterInfo value)
- throws YarnException {
+ private long getAvailableMemory(SubClusterInfo value) throws YarnException {
try {
long mem = -1;
JSONObject obj = new JSONObject(value.getCapability());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
index a8ac5f7..bc3a1f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -17,39 +17,32 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import java.util.Map;
-
/**
* This implements a policy that interprets "weights" as a ordered list of
* preferences among sub-clusters. Highest weight among active subclusters is
* chosen.
*/
-public class PriorityRouterPolicy
- extends BaseWeightedRouterPolicy {
-
- private static final Log LOG =
- LogFactory.getLog(PriorityRouterPolicy.class);
+public class PriorityRouterPolicy extends AbstractRouterPolicy {
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
// This finds the sub-cluster with the highest weight among the
// currently active ones.
- Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
- .getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> weights =
+ getPolicyInfo().getRouterPolicyWeights();
SubClusterId chosen = null;
Float currentBest = Float.MIN_VALUE;
for (SubClusterId id : activeSubclusters.keySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
index 1774961..b8f9cc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -17,6 +17,11 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
@@ -25,11 +30,6 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPo
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
/**
* This simple policy picks at uniform random among any of the currently active
* subclusters. This policy is easy to use and good for testing.
@@ -39,7 +39,7 @@ import java.util.Random;
* of the "weights", in which case the {@link UniformRandomRouterPolicy} send
* load to them, while {@code WeightedRandomRouterPolicy} does not.
*/
-public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
+public class UniformRandomRouterPolicy extends AbstractRouterPolicy {
private Random rand;
@@ -49,14 +49,14 @@ public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
@Override
public void reinitialize(
- FederationPolicyInitializationContext federationPolicyContext)
+ FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
FederationPolicyInitializationContextValidator
- .validate(federationPolicyContext, this.getClass().getCanonicalName());
+ .validate(policyContext, this.getClass().getCanonicalName());
- //note: this overrides BaseWeighterRouterPolicy and ignores the weights
+ // note: this overrides AbstractRouterPolicy and ignores the weights
- setPolicyContext(federationPolicyContext);
+ setPolicyContext(policyContext);
}
/**
@@ -64,21 +64,19 @@ public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
* depend on the weights in the policy).
*
* @param appSubmissionContext the context for the app being submitted
- * (ignored).
+ * (ignored).
*
* @return a randomly chosen subcluster.
*
* @throws YarnException if there are no active subclusters.
*/
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
- List<SubClusterId> list =
- new ArrayList<>(activeSubclusters.keySet());
+ List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
return list.get(rand.nextInt(list.size()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
index 0777677..ac75ae9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -18,32 +18,30 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+import java.util.Random;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-
-import java.util.Map;
-import java.util.Random;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This policy implements a weighted random sample among currently active
* sub-clusters.
*/
-public class WeightedRandomRouterPolicy
- extends BaseWeightedRouterPolicy {
+public class WeightedRandomRouterPolicy extends AbstractRouterPolicy {
- private static final Log LOG =
- LogFactory.getLog(WeightedRandomRouterPolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WeightedRandomRouterPolicy.class);
private Random rand = new Random(System.currentTimeMillis());
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
@@ -52,13 +50,13 @@ public class WeightedRandomRouterPolicy
// changes dynamically (and this would unfairly spread the load to
// sub-clusters adjacent to an inactive one), hence we need to count/scan
// the list and based on weight pick the next sub-cluster.
- Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
- .getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> weights =
+ getPolicyInfo().getRouterPolicyWeights();
float totActiveWeight = 0;
- for(Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()){
- if(entry.getKey()!=null && activeSubclusters.containsKey(entry.getKey()
- .toId())){
+ for (Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()) {
+ if (entry.getKey() != null
+ && activeSubclusters.containsKey(entry.getKey().toId())) {
totActiveWeight += entry.getValue();
}
}
@@ -73,7 +71,7 @@ public class WeightedRandomRouterPolicy
return id;
}
}
- //should never happen
+ // should never happen
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
index 5d0fcb6..e445ac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
@@ -17,4 +17,3 @@
*/
/** Router policies. **/
package org.apache.hadoop.yarn.server.federation.policies.router;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
index 8238633..6b4f60c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
@@ -57,11 +57,11 @@ public abstract class AbstractSubClusterResolver implements SubClusterResolver {
return rackToSubClusters.get(rackname);
}
- protected Map<String, SubClusterId> getNodeToSubCluster() {
+ public Map<String, SubClusterId> getNodeToSubCluster() {
return nodeToSubCluster;
}
- protected Map<String, Set<SubClusterId>> getRackToSubClusters() {
+ public Map<String, Set<SubClusterId>> getRackToSubClusters() {
return rackToSubClusters;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
index 8da92b9..ba897da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -22,14 +22,17 @@ import static org.mockito.Mockito.mock;
import java.nio.ByteBuffer;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -49,6 +52,7 @@ public abstract class BaseFederationPoliciesTest {
private ApplicationSubmissionContext applicationSubmissionContext =
mock(ApplicationSubmissionContext.class);
private Random rand = new Random();
+ private SubClusterId homeSubCluster;
@Test
public void testReinitilialize() throws YarnException {
@@ -88,16 +92,22 @@ public abstract class BaseFederationPoliciesTest {
getPolicy().reinitialize(fpc);
}
- @Test(expected = NoActiveSubclustersException.class)
+ @Test(expected = FederationPolicyException.class)
public void testNoSubclusters() throws YarnException {
// empty the activeSubclusters map
FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
getPolicyInfo(), new HashMap<>());
- ConfigurableFederationPolicy currentPolicy = getPolicy();
- if (currentPolicy instanceof FederationRouterPolicy) {
- ((FederationRouterPolicy) currentPolicy)
+ ConfigurableFederationPolicy localPolicy = getPolicy();
+ if (localPolicy instanceof FederationRouterPolicy) {
+ ((FederationRouterPolicy) localPolicy)
.getHomeSubcluster(getApplicationSubmissionContext());
+ } else {
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+ ((FederationAMRMProxyPolicy) localPolicy)
+ .splitResourceRequests(resourceRequests);
}
}
@@ -152,4 +162,12 @@ public abstract class BaseFederationPoliciesTest {
this.rand = rand;
}
+ public SubClusterId getHomeSubCluster() {
+ return homeSubCluster;
+ }
+
+ public void setHomeSubCluster(SubClusterId homeSubCluster) {
+ this.homeSubCluster = homeSubCluster;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index e840b3f..c79fd2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -16,22 +16,20 @@
* limitations under the License.
*/
-
package org.apache.hadoop.yarn.server.federation.policies;
+import java.nio.ByteBuffer;
+
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.junit.Before;
import org.junit.Test;
-import java.nio.ByteBuffer;
-
/**
* Test class for {@link FederationPolicyInitializationContextValidator}.
*/
@@ -45,11 +43,10 @@ public class TestFederationPolicyInitializationContextValidator {
@Before
public void setUp() throws Exception {
goodFacade = FederationPoliciesTestUtil.initFacade();
- goodConfig =
- new MockPolicyManager().serializeConf();
- goodSR =FederationPoliciesTestUtil.initResolver();
- context = new
- FederationPolicyInitializationContext(goodConfig, goodSR, goodFacade);
+ goodConfig = new MockPolicyManager().serializeConf();
+ goodSR = FederationPoliciesTestUtil.initResolver();
+ context = new FederationPolicyInitializationContext(goodConfig, goodSR,
+ goodFacade);
}
@Test
@@ -100,8 +97,7 @@ public class TestFederationPolicyInitializationContextValidator {
@Override
public FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationAMRMProxyPolicy oldInstance)
throws FederationPolicyInitializationException {
return null;
@@ -109,8 +105,7 @@ public class TestFederationPolicyInitializationContextValidator {
@Override
public FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationRouterPolicy oldInstance)
throws FederationPolicyInitializationException {
return null;
@@ -120,8 +115,8 @@ public class TestFederationPolicyInitializationContextValidator {
public SubClusterPolicyConfiguration serializeConf()
throws FederationPolicyInitializationException {
ByteBuffer buf = ByteBuffer.allocate(0);
- return SubClusterPolicyConfiguration
- .newInstance("queue1", this.getClass().getCanonicalName(), buf);
+ return SubClusterPolicyConfiguration.newInstance("queue1",
+ this.getClass().getCanonicalName(), buf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
new file mode 100644
index 0000000..a21f53d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link BroadcastAMRMProxyPolicy}.
+ */
+public class TestBroadcastAMRMProxyFederationPolicy
+ extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new BroadcastAMRMProxyPolicy());
+ // needed for base test to work
+ setPolicyInfo(mock(WeightedPolicyInfo.class));
+
+ for (int i = 1; i <= 2; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ mock(WeightedPolicyInfo.class), getActiveSubclusters());
+
+ }
+
+ @Test
+ public void testSplitAllocateRequest() throws Exception {
+ // verify the request is broadcasted to all subclusters
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+ Assert.assertTrue(response.size() == 2);
+ for (Map.Entry<SubClusterId, List<ResourceRequest>> entry : response
+ .entrySet()) {
+ Assert.assertTrue(getActiveSubclusters().get(entry.getKey()) != null);
+ for (ResourceRequest r : entry.getValue()) {
+ Assert.assertTrue(resourceRequests.contains(r));
+ }
+ }
+ for (SubClusterId subClusterId : getActiveSubclusters().keySet()) {
+ for (ResourceRequest r : response.get(subClusterId)) {
+ Assert.assertTrue(resourceRequests.contains(r));
+ }
+ }
+ }
+
+ @Test
+ public void testNotifyOfResponse() throws Exception {
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ try {
+ ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
+ SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));
+ Assert.fail();
+ } catch (FederationPolicyException f) {
+ System.out.println("Expected: " + f.getMessage());
+ }
+
+ ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
+ SubClusterId.newInstance("sc1"), mock(AllocateResponse.class));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
new file mode 100644
index 0000000..2654a06
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple test class for the {@link LocalityMulticastAMRMProxyPolicy}.
+ */
+public class TestLocalityMulticastAMRMProxyPolicy
+ extends BaseFederationPoliciesTest {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestLocalityMulticastAMRMProxyPolicy.class);
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new LocalityMulticastAMRMProxyPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 subclusters with a 5% chance of being inactive
+ for (int i = 0; i < 6; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("subcluster" + i);
+ // sub-cluster 3 is not active
+ if (i != 3) {
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ float weight = 1 / 10f;
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ // sub-cluster 4 is "disabled" in the weights
+ if (i == 4) {
+ routerWeights.put(sc, 0f);
+ amrmWeights.put(sc, 0f);
+ }
+ }
+
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+ getPolicyInfo().setHeadroomAlpha(0.5f);
+ setHomeSubCluster(SubClusterId.newInstance("homesubcluster"));
+
+ }
+
+ @Test
+ public void testReinitilialize() throws YarnException {
+ initializePolicy();
+ }
+
+ private void initializePolicy() throws YarnException {
+ setFederationPolicyContext(new FederationPolicyInitializationContext());
+ SubClusterResolver resolver = FederationPoliciesTestUtil.initResolver();
+ getFederationPolicyContext().setFederationSubclusterResolver(resolver);
+ ByteBuffer buf = getPolicyInfo().toByteBuffer();
+ getFederationPolicyContext().setSubClusterPolicyConfiguration(
+ SubClusterPolicyConfiguration.newInstance("queue1",
+ getPolicy().getClass().getCanonicalName(), buf));
+ getFederationPolicyContext().setHomeSubcluster(getHomeSubCluster());
+ FederationPoliciesTestUtil.initializePolicyContext(
+ getFederationPolicyContext(), getPolicy(), getPolicyInfo(),
+ getActiveSubclusters());
+ }
+
+ @Test
+ public void testSplitBasedOnHeadroom() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+ // Configure policy to be 100% headroom based
+ getPolicyInfo().setHeadroomAlpha(1.0f);
+
+ initializePolicy();
+ List<ResourceRequest> resourceRequests = createSimpleRequest();
+
+ prepPolicyWithHeadroom();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ // pretty print requests
+ LOG.info("Initial headroom");
+ prettyPrintRequests(response);
+
+ validateSplit(response, resourceRequests);
+
+ // based on headroom, we expect 75 containers to got to subcluster0,
+ // as it advertise lots of headroom (100), no containers for sublcuster1
+ // as it advertise zero headroom, 1 to subcluster 2 (as it advertise little
+ // headroom (1), and 25 to subcluster5 which has unknown headroom, and so
+ // it gets 1/4th of the load
+ checkExpectedAllocation(response, "subcluster0", 1, 75);
+ checkExpectedAllocation(response, "subcluster1", 1, -1);
+ checkExpectedAllocation(response, "subcluster2", 1, 1);
+ checkExpectedAllocation(response, "subcluster5", 1, 25);
+
+ // notify a change in headroom and try again
+ AllocateResponse ar = getAllocateResponseWithTargetHeadroom(100);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster2"), ar);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ LOG.info("After headroom update");
+ prettyPrintRequests(response);
+ validateSplit(response, resourceRequests);
+
+ // we simulated a change in headroom for subcluster2, which will now
+ // have the same headroom of subcluster0 and so it splits the requests
+ // note that the total is still less or equal to (userAsk + numSubClusters)
+ checkExpectedAllocation(response, "subcluster0", 1, 38);
+ checkExpectedAllocation(response, "subcluster1", 1, -1);
+ checkExpectedAllocation(response, "subcluster2", 1, 38);
+ checkExpectedAllocation(response, "subcluster5", 1, 25);
+
+ }
+
+ @Test(timeout = 5000)
+ public void testStressPolicy() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+ // Configure policy to be 100% headroom based
+ getPolicyInfo().setHeadroomAlpha(1.0f);
+
+ initializePolicy();
+
+ int numRR = 1000;
+ List<ResourceRequest> resourceRequests = createLargeRandomList(numRR);
+
+ prepPolicyWithHeadroom();
+
+ int numIterations = 1000;
+ long tstart = System.currentTimeMillis();
+ for (int i = 0; i < numIterations; i++) {
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+ validateSplit(response, resourceRequests);
+ }
+ long tend = System.currentTimeMillis();
+
+ LOG.info("Performed " + numIterations + " policy invocations (and "
+ + "validations) in " + (tend - tstart) + "ms");
+ }
+
+ @Test
+ public void testFWDAllZeroANY() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+ // Configure policy to be 100% headroom based
+ getPolicyInfo().setHeadroomAlpha(0.5f);
+
+ initializePolicy();
+ List<ResourceRequest> resourceRequests = createZeroSizedANYRequest();
+
+ // this receives responses from sc0,sc1,sc2
+ prepPolicyWithHeadroom();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ // we expect all three to appear for a zero-sized ANY
+
+ // pretty print requests
+ prettyPrintRequests(response);
+
+ validateSplit(response, resourceRequests);
+
+ // we expect the zero size request to be sent to the first 3 rm (due to
+ // the fact that we received responses only from these 3 sublcusters)
+ checkExpectedAllocation(response, "subcluster0", 1, 0);
+ checkExpectedAllocation(response, "subcluster1", 1, 0);
+ checkExpectedAllocation(response, "subcluster2", 1, 0);
+ checkExpectedAllocation(response, "subcluster3", -1, -1);
+ checkExpectedAllocation(response, "subcluster4", -1, -1);
+ checkExpectedAllocation(response, "subcluster5", -1, -1);
+ }
+
+ @Test
+ public void testSplitBasedOnHeadroomAndWeights() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+
+ // Configure policy to be 50% headroom based and 50% weight based
+ getPolicyInfo().setHeadroomAlpha(0.5f);
+
+ initializePolicy();
+ List<ResourceRequest> resourceRequests = createSimpleRequest();
+
+ prepPolicyWithHeadroom();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ // pretty print requests
+ prettyPrintRequests(response);
+
+ validateSplit(response, resourceRequests);
+
+ // in this case the headroom allocates 50 containers, while weights allocate
+ // the rest. due to weights we have 12.5 (round to 13) containers for each
+ // sublcuster, the rest is due to headroom.
+ checkExpectedAllocation(response, "subcluster0", 1, 50);
+ checkExpectedAllocation(response, "subcluster1", 1, 13);
+ checkExpectedAllocation(response, "subcluster2", 1, 13);
+ checkExpectedAllocation(response, "subcluster3", -1, -1);
+ checkExpectedAllocation(response, "subcluster4", -1, -1);
+ checkExpectedAllocation(response, "subcluster5", 1, 25);
+
+ }
+
+ private void prepPolicyWithHeadroom() throws YarnException {
+ AllocateResponse ar = getAllocateResponseWithTargetHeadroom(100);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster0"), ar);
+
+ ar = getAllocateResponseWithTargetHeadroom(0);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster1"), ar);
+
+ ar = getAllocateResponseWithTargetHeadroom(1);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster2"), ar);
+ }
+
+ private AllocateResponse getAllocateResponseWithTargetHeadroom(
+ int numContainers) {
+ return AllocateResponse.newInstance(0, null, null,
+ Collections.<NodeReport> emptyList(),
+ Resource.newInstance(numContainers * 1024, numContainers), null, 10,
+ null, Collections.<NMToken> emptyList());
+ }
+
+ @Test
+ public void testSplitAllocateRequest() throws Exception {
+
+ // Test a complex List<ResourceRequest> is split correctly
+ initializePolicy();
+
+ // modify default initialization to include a "homesubcluster"
+ // which we will use as the default for when nodes or racks are unknown
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(getHomeSubCluster());
+ getActiveSubclusters().put(getHomeSubCluster(), sci);
+ SubClusterIdInfo sc = new SubClusterIdInfo(getHomeSubCluster().getId());
+
+ getPolicyInfo().getRouterPolicyWeights().put(sc, 0.1f);
+ getPolicyInfo().getAMRMPolicyWeights().put(sc, 0.1f);
+
+ FederationPoliciesTestUtil.initializePolicyContext(
+ getFederationPolicyContext(), getPolicy(), getPolicyInfo(),
+ getActiveSubclusters());
+
+ List<ResourceRequest> resourceRequests = createComplexRequest();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ validateSplit(response, resourceRequests);
+ prettyPrintRequests(response);
+
+ // we expect 4 entry for home subcluster (3 for request-id 4, and a part
+ // of the broadcast of request-id 2
+ checkExpectedAllocation(response, getHomeSubCluster().getId(), 4, 23);
+
+ // for subcluster0 we expect 3 entry from request-id 0, and 3 from
+ // request-id 3, as well as part of the request-id 2 broadast
+ checkExpectedAllocation(response, "subcluster0", 7, 26);
+
+ // we expect 5 entry for subcluster1 (4 from request-id 1, and part
+ // of the broadcast of request-id 2
+ checkExpectedAllocation(response, "subcluster1", 5, 25);
+
+ // sub-cluster 2 should contain 3 entry from request-id 1 and 1 from the
+ // broadcast of request-id 2, and no request-id 0
+ checkExpectedAllocation(response, "subcluster2", 4, 23);
+
+ // subcluster id 3, 4 should not appear (due to weights or active/inactive)
+ checkExpectedAllocation(response, "subcluster3", -1, -1);
+ checkExpectedAllocation(response, "subcluster4", -1, -1);
+
+ // subcluster5 should get only part of the request-id 2 broadcast
+ checkExpectedAllocation(response, "subcluster5", 1, 20);
+
+ // check that the allocations that show up are what expected
+ for (ResourceRequest rr : response.get(getHomeSubCluster())) {
+ Assert.assertTrue(rr.getAllocationRequestId() == 4L
+ || rr.getAllocationRequestId() == 2L);
+ }
+
+ for (ResourceRequest rr : response.get(getHomeSubCluster())) {
+ Assert.assertTrue(rr.getAllocationRequestId() != 1L);
+ }
+
+ List<ResourceRequest> rrs =
+ response.get(SubClusterId.newInstance("subcluster0"));
+ for (ResourceRequest rr : rrs) {
+ Assert.assertTrue(rr.getAllocationRequestId() != 1L);
+ }
+
+ for (ResourceRequest rr : response
+ .get(SubClusterId.newInstance("subcluster2"))) {
+ Assert.assertTrue(rr.getAllocationRequestId() != 0L);
+ }
+
+ for (ResourceRequest rr : response
+ .get(SubClusterId.newInstance("subcluster5"))) {
+ Assert.assertTrue(rr.getAllocationRequestId() >= 2);
+ Assert.assertTrue(rr.getRelaxLocality());
+ }
+ }
+
+ // check that the number of containers in the first ResourceRequest in
+ // response for this sub-cluster matches expectations. -1 indicate the
+ // response should be null
+ private void checkExpectedAllocation(
+ Map<SubClusterId, List<ResourceRequest>> response, String subCluster,
+ long totResourceRequests, long totContainers) {
+ if (totContainers == -1) {
+ Assert.assertNull(response.get(SubClusterId.newInstance(subCluster)));
+ } else {
+ SubClusterId sc = SubClusterId.newInstance(subCluster);
+ Assert.assertEquals(totResourceRequests, response.get(sc).size());
+
+ long actualContCount = 0;
+ for (ResourceRequest rr : response.get(sc)) {
+ actualContCount += rr.getNumContainers();
+ }
+ Assert.assertEquals(totContainers, actualContCount);
+ }
+ }
+
+ private void validateSplit(Map<SubClusterId, List<ResourceRequest>> split,
+ List<ResourceRequest> original) throws YarnException {
+
+ SubClusterResolver resolver =
+ getFederationPolicyContext().getFederationSubclusterResolver();
+
+ // Apply general validation rules
+ int numUsedSubclusters = split.size();
+
+ Set<Long> originalIds = new HashSet<>();
+ Set<Long> splitIds = new HashSet<>();
+
+ int originalContainers = 0;
+ for (ResourceRequest rr : original) {
+ originalContainers += rr.getNumContainers();
+ originalIds.add(rr.getAllocationRequestId());
+ }
+
+ int splitContainers = 0;
+ for (Map.Entry<SubClusterId, List<ResourceRequest>> rrs : split
+ .entrySet()) {
+ for (ResourceRequest rr : rrs.getValue()) {
+ splitContainers += rr.getNumContainers();
+ splitIds.add(rr.getAllocationRequestId());
+ // check node-local asks are sent to right RM (only)
+ SubClusterId fid = null;
+ try {
+ fid = resolver.getSubClusterForNode(rr.getResourceName());
+ } catch (YarnException e) {
+ // ignore code will handle
+ }
+ if (!rrs.getKey().equals(getHomeSubCluster()) && fid != null
+ && !fid.equals(rrs.getKey())) {
+ Assert.fail("A node-local (or resolvable rack-local) RR should not "
+ + "be send to an RM other than what it resolves to.");
+ }
+ }
+ }
+
+ // check we are not inventing Allocation Ids
+ Assert.assertEquals(originalIds, splitIds);
+
+ // check we are not exceedingly replicating the container asks among
+ // RMs (a little is allowed due to rounding of fractional splits)
+ Assert.assertTrue(
+ " Containers requested (" + splitContainers + ") should "
+ + "not exceed the original count of containers ("
+ + originalContainers + ") by more than the number of subclusters ("
+ + numUsedSubclusters + ")",
+ originalContainers + numUsedSubclusters >= splitContainers);
+
+ // Test target Ids
+ for (SubClusterId targetId : split.keySet()) {
+ Assert.assertTrue("Target subclusters should be in the active set",
+ getActiveSubclusters().containsKey(targetId));
+ Assert.assertTrue(
+ "Target subclusters (" + targetId + ") should have weight >0 in "
+ + "the policy ",
+ getPolicyInfo().getRouterPolicyWeights()
+ .get(new SubClusterIdInfo(targetId)) > 0);
+ }
+ }
+
+ private void prettyPrintRequests(
+ Map<SubClusterId, List<ResourceRequest>> response) {
+ for (Map.Entry<SubClusterId, List<ResourceRequest>> entry : response
+ .entrySet()) {
+ String str = "";
+ for (ResourceRequest rr : entry.getValue()) {
+ str += " [id:" + rr.getAllocationRequestId() + " loc:"
+ + rr.getResourceName() + " numCont:" + rr.getNumContainers()
+ + "], ";
+ }
+ LOG.info(entry.getKey() + " --> " + str);
+ }
+ }
+
+ private List<ResourceRequest> createLargeRandomList(int numRR)
+ throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+ Random rand = new Random(1);
+ DefaultSubClusterResolverImpl resolver =
+ (DefaultSubClusterResolverImpl) getFederationPolicyContext()
+ .getFederationSubclusterResolver();
+
+ List<String> nodes =
+ new ArrayList<>(resolver.getNodeToSubCluster().keySet());
+
+ for (int i = 0; i < numRR; i++) {
+ String nodeName = nodes.get(rand.nextInt(nodes.size()));
+ long allocationId = (long) rand.nextInt(20);
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(allocationId,
+ nodeName, 1024, 1, 1, rand.nextInt(100), null, rand.nextBoolean()));
+ }
+ return out;
+ }
+
+ private List<ResourceRequest> createSimpleRequest() throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ ResourceRequest.ANY, 1024, 1, 1, 100, null, true));
+ return out;
+ }
+
+ private List<ResourceRequest> createZeroSizedANYRequest() throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ ResourceRequest.ANY, 1024, 1, 1, 0, null, true));
+ return out;
+ }
+
+ private List<ResourceRequest> createComplexRequest() throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ "subcluster0-rack0-host0", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ "subcluster0-rack0", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ ResourceRequest.ANY, 1024, 1, 1, 1, null, false));
+
+ // create a single container request with 3 alternative hosts across sc1,sc2
+ // where we want 2 containers in sc1 and 1 in sc2
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster1-rack1-host1", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster1-rack1-host2", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster2-rack3-host3", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster1-rack1", 1024, 1, 1, 2, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster2-rack3", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ ResourceRequest.ANY, 1024, 1, 1, 2, null, false));
+
+ // create a non-local ANY request that can span anything
+ out.add(FederationPoliciesTestUtil.createResourceRequest(2L,
+ ResourceRequest.ANY, 1024, 1, 1, 100, null, true));
+
+ // create a single container request in sc0 with relaxed locality
+ out.add(FederationPoliciesTestUtil.createResourceRequest(3L,
+ "subcluster0-rack0-host0", 1024, 1, 1, 1, null, true));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(3L,
+ "subcluster0-rack0", 1024, 1, 1, 1, null, true));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(3L,
+ ResourceRequest.ANY, 1024, 1, 1, 1, null, true));
+
+ // create a request of an unknown node/rack and expect this to show up
+ // in homesubcluster
+ out.add(FederationPoliciesTestUtil.createResourceRequest(4L, "unknownNode",
+ 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(4L, "unknownRack",
+ 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(4L,
+ ResourceRequest.ANY, 1024, 1, 1, 1, null, false));
+
+ return out;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index 9e94f72..906e35f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
@@ -29,12 +32,9 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-
/**
- * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the
- * load is properly considered for allocation.
+ * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load
+ * is properly considered for allocation.
*/
public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
@@ -47,12 +47,10 @@ public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
// simulate 20 active subclusters
for (int i = 0; i < 20; i++) {
- SubClusterIdInfo sc =
- new SubClusterIdInfo(String.format("sc%02d", i));
+ SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", i));
SubClusterInfo federationSubClusterInfo =
SubClusterInfo.newInstance(sc.toId(), null, null, null, null, -1,
- SubClusterState.SC_RUNNING, -1,
- generateClusterMetricsInfo(i));
+ SubClusterState.SC_RUNNING, -1, generateClusterMetricsInfo(i));
getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
float weight = getRand().nextInt(2);
if (i == 5) {
@@ -76,7 +74,7 @@ public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
private String generateClusterMetricsInfo(int id) {
long mem = 1024 * getRand().nextInt(277 * 100 - 1);
- //plant a best cluster
+ // plant a best cluster
if (id == 5) {
mem = 1024 * 277 * 100;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
index ff5175d..eefcfd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -16,6 +16,12 @@
*/
package org.apache.hadoop.yarn.server.federation.policies.router;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
@@ -28,12 +34,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
/**
* Simple test class for the {@link PriorityRouterPolicy}. Tests that the
* weights are correctly used for ordering the choice of sub-clusters.
@@ -72,8 +72,7 @@ public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest {
getPolicyInfo().setRouterPolicyWeights(routerWeights);
getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
- getPolicyInfo(),
- getActiveSubclusters());
+ getPolicyInfo(), getActiveSubclusters());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
index a612685..78967d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -17,6 +17,13 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
@@ -29,13 +36,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
/**
* Simple test class for the {@link WeightedRandomRouterPolicy}. Generate large
* number of randomized tests to check we are weighiting correctly even if
@@ -71,8 +71,7 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
- getPolicyInfo(),
- getActiveSubclusters());
+ getPolicyInfo(), getActiveSubclusters());
}
@@ -88,8 +87,8 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
float numberOfDraws = 1000000;
for (float i = 0; i < numberOfDraws; i++) {
- SubClusterId chosenId = ((FederationRouterPolicy) getPolicy()).
- getHomeSubcluster(getApplicationSubmissionContext());
+ SubClusterId chosenId = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
counter.get(chosenId).incrementAndGet();
}
@@ -113,13 +112,15 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
if (getActiveSubclusters().containsKey(counterEntry.getKey())) {
Assert.assertTrue(
"Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
- + " expected weight: " + expectedWeight, expectedWeight == 0 ||
- (actualWeight / expectedWeight) < 1.1
- && (actualWeight / expectedWeight) > 0.9);
+ + " expected weight: " + expectedWeight,
+ expectedWeight == 0 || (actualWeight / expectedWeight) < 1.1
+ && (actualWeight / expectedWeight) > 0.9);
} else {
- Assert.assertTrue(
- "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
- + " expected weight: " + expectedWeight, actualWeight == 0);
+ Assert
+ .assertTrue(
+ "Id " + counterEntry.getKey() + " Actual weight: "
+ + actualWeight + " expected weight: " + expectedWeight,
+ actualWeight == 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index f901329..87ed8d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.server.federation.utils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
@@ -26,6 +27,7 @@ import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolv
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.*;
+import org.apache.hadoop.yarn.util.Records;
import java.net.URL;
import java.nio.ByteBuffer;
@@ -48,6 +50,68 @@ public final class FederationPoliciesTestUtil {
// disabled.
}
+ private static final String FEDR_NODE_PREFIX = "fedr-test-node-";
+
+
+ public static List<ResourceRequest> createResourceRequests(String[] hosts,
+ int memory, int vCores, int priority, int containers,
+ String labelExpression, boolean relaxLocality) throws YarnException {
+ List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
+ for (String host : hosts) {
+ ResourceRequest hostReq =
+ createResourceRequest(host, memory, vCores, priority, containers,
+ labelExpression, relaxLocality);
+ reqs.add(hostReq);
+ ResourceRequest rackReq =
+ createResourceRequest("/default-rack", memory, vCores, priority,
+ containers, labelExpression, relaxLocality);
+ reqs.add(rackReq);
+ }
+
+ ResourceRequest offRackReq =
+ createResourceRequest(ResourceRequest.ANY, memory, vCores, priority,
+ containers, labelExpression, relaxLocality);
+ reqs.add(offRackReq);
+ return reqs;
+ }
+
+ protected static ResourceRequest createResourceRequest(String resource,
+ int memory, int vCores, int priority, int containers,
+ boolean relaxLocality) throws YarnException {
+ return createResourceRequest(resource, memory, vCores, priority, containers,
+ null, relaxLocality);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public static ResourceRequest createResourceRequest(long id, String resource,
+ int memory, int vCores, int priority, int containers,
+ String labelExpression, boolean relaxLocality) throws YarnException {
+ ResourceRequest out =
+ createResourceRequest(resource, memory, vCores, priority, containers,
+ labelExpression, relaxLocality);
+ out.setAllocationRequestId(id);
+ return out;
+ }
+
+ public static ResourceRequest createResourceRequest(String resource,
+ int memory, int vCores, int priority, int containers,
+ String labelExpression, boolean relaxLocality) throws YarnException {
+ ResourceRequest req = Records.newRecord(ResourceRequest.class);
+ req.setResourceName(resource);
+ req.setNumContainers(containers);
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(priority);
+ req.setPriority(pri);
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemorySize(memory);
+ capability.setVirtualCores(vCores);
+ req.setCapability(capability);
+ if (labelExpression != null) {
+ req.setNodeLabelExpression(labelExpression);
+ }
+ req.setRelaxLocality(relaxLocality);
+ return req;
+ }
public static void initializePolicyContext(
FederationPolicyInitializationContext fpc, ConfigurableFederationPolicy
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
index e4d6112..2b7e237 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
@@ -1,4 +1,8 @@
node1,subcluster1,rack1
node2 , subcluster2, RACK1
noDE3,subcluster3, rack2
-node4, subcluster3, rack2
\ No newline at end of file
+node4, subcluster3, rack2
+subcluster0-rack0-host0,subcluster0, subcluster0-rack0
+Subcluster1-RACK1-HOST1,subcluster1, subCluster1-RACK1
+SUBCLUSTER1-RACK1-HOST2,subcluster1, subCluster1-RACK1
+SubCluster2-RACK3-HOST3,subcluster2, subcluster2-rack3
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-5407. In-memory based
implementation of the
FederationApplicationStateStore/FederationPolicyStateStore. (Ellen Hui via
Subru)
Posted by su...@apache.org.
YARN-5407. In-memory based implementation of the FederationApplicationStateStore/FederationPolicyStateStore. (Ellen Hui via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed2c6c79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed2c6c79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed2c6c79
Branch: refs/heads/YARN-2915
Commit: ed2c6c791ed1190e9ef4a8fc544f17af3802f2a6
Parents: 630cb6c
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Aug 9 16:07:55 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../store/impl/MemoryFederationStateStore.java | 158 +++++++-
...SubClusterPoliciesConfigurationsRequest.java | 2 +-
...ubClusterPoliciesConfigurationsResponse.java | 2 +-
...GetSubClusterPolicyConfigurationRequest.java | 3 +-
...etSubClusterPolicyConfigurationResponse.java | 2 +-
...SetSubClusterPolicyConfigurationRequest.java | 20 +-
...etSubClusterPolicyConfigurationResponse.java | 2 +-
.../records/SubClusterPolicyConfiguration.java | 27 +-
...tApplicationHomeSubClusterRequestPBImpl.java | 4 +
...ClusterPolicyConfigurationRequestPBImpl.java | 17 -
.../pb/SubClusterPolicyConfigurationPBImpl.java | 17 +
.../proto/yarn_server_federation_protos.proto | 8 +-
.../impl/FederationStateStoreBaseTest.java | 367 ++++++++++++++++++-
.../impl/TestMemoryFederationStateStore.java | 4 +-
14 files changed, 558 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index cea4ac2..a540dff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -20,35 +20,72 @@ package org.apache.hadoop.yarn.server.federation.store.impl;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
/**
- * In-memory implementation of FederationMembershipStateStore.
+ * In-memory implementation of {@link FederationStateStore}.
*/
-public class MemoryFederationStateStore
- implements FederationMembershipStateStore {
+public class MemoryFederationStateStore implements FederationStateStore {
+
+ private Map<SubClusterId, SubClusterInfo> membership;
+ private Map<ApplicationId, SubClusterId> applications;
+ private Map<String, SubClusterPolicyConfiguration> policies;
- private final Map<SubClusterId, SubClusterInfo> membership =
- new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
private final MonotonicClock clock = new MonotonicClock();
@Override
+ public void init(Configuration conf) {
+ membership = new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
+ applications = new ConcurrentHashMap<ApplicationId, SubClusterId>();
+ policies = new ConcurrentHashMap<String, SubClusterPolicyConfiguration>();
+ }
+
+ @Override
+ public void close() {
+ membership = null;
+ applications = null;
+ policies = null;
+ }
+
+ @Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
@@ -116,4 +153,113 @@ public class MemoryFederationStateStore
return GetSubClustersInfoResponse.newInstance(result);
}
+ // FederationApplicationHomeSubClusterStore methods
+
+ @Override
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId =
+ request.getApplicationHomeSubCluster().getApplicationId();
+ if (applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " already exists");
+ }
+
+ applications.put(appId,
+ request.getApplicationHomeSubCluster().getHomeSubCluster());
+ return AddApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId =
+ request.getApplicationHomeSubCluster().getApplicationId();
+ if (!applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " does not exist");
+ }
+
+ applications.put(appId,
+ request.getApplicationHomeSubCluster().getHomeSubCluster());
+ return UpdateApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId = request.getApplicationId();
+ if (!applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " does not exist");
+ }
+
+ return GetApplicationHomeSubClusterResponse.newInstance(
+ ApplicationHomeSubCluster.newInstance(appId, applications.get(appId)));
+ }
+
+ @Override
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException {
+ List<ApplicationHomeSubCluster> result =
+ new ArrayList<ApplicationHomeSubCluster>();
+ for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
+ result
+ .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+ }
+
+ GetApplicationsHomeSubClusterResponse.newInstance(result);
+ return GetApplicationsHomeSubClusterResponse.newInstance(result);
+ }
+
+ @Override
+ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ DeleteApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId = request.getApplicationId();
+ if (!applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " does not exist");
+ }
+
+ applications.remove(appId);
+ return DeleteApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ String queue = request.getQueue();
+ if (!policies.containsKey(queue)) {
+ throw new YarnException("Policy for queue " + queue + " does not exist");
+ }
+
+ return GetSubClusterPolicyConfigurationResponse
+ .newInstance(policies.get(queue));
+ }
+
+ @Override
+ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ policies.put(request.getPolicyConfiguration().getQueue(),
+ request.getPolicyConfiguration());
+ return SetSubClusterPolicyConfigurationResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException {
+ ArrayList<SubClusterPolicyConfiguration> result =
+ new ArrayList<SubClusterPolicyConfiguration>();
+ for (SubClusterPolicyConfiguration policy : policies.values()) {
+ result.add(policy);
+ }
+ return GetSubClusterPoliciesConfigurationsResponse.newInstance(result);
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return null;
+ }
+
+ @Override
+ public Version loadVersion() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
index 404521b..8cb84f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.util.Records;
@Private
@Unstable
public abstract class GetSubClusterPoliciesConfigurationsRequest {
- public GetSubClusterPoliciesConfigurationsRequest newInstance() {
+ public static GetSubClusterPoliciesConfigurationsRequest newInstance() {
return Records.newRecord(GetSubClusterPoliciesConfigurationsRequest.class);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
index 6554d68..2eaeb51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
@@ -36,7 +36,7 @@ public abstract class GetSubClusterPoliciesConfigurationsResponse {
@Private
@Unstable
- public GetSubClusterPoliciesConfigurationsResponse newInstance(
+ public static GetSubClusterPoliciesConfigurationsResponse newInstance(
List<SubClusterPolicyConfiguration> policyConfigurations) {
GetSubClusterPoliciesConfigurationsResponse response =
Records.newRecord(GetSubClusterPoliciesConfigurationsResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
index 7b7d8c4..c3f49e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
@@ -33,7 +33,8 @@ public abstract class GetSubClusterPolicyConfigurationRequest {
@Private
@Unstable
- public GetSubClusterPolicyConfigurationRequest newInstance(String queueName) {
+ public static GetSubClusterPolicyConfigurationRequest newInstance(
+ String queueName) {
GetSubClusterPolicyConfigurationRequest request =
Records.newRecord(GetSubClusterPolicyConfigurationRequest.class);
request.setQueue(queueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
index 11a46e0..350b239 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
@@ -34,7 +34,7 @@ public abstract class GetSubClusterPolicyConfigurationResponse {
@Private
@Unstable
- public GetSubClusterPolicyConfigurationResponse newInstance(
+ public static GetSubClusterPolicyConfigurationResponse newInstance(
SubClusterPolicyConfiguration policy) {
GetSubClusterPolicyConfigurationResponse response =
Records.newRecord(GetSubClusterPolicyConfigurationResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
index 06d5399..743ad0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class SetSubClusterPolicyConfigurationRequest {
@Private
@Unstable
- public SetSubClusterPolicyConfigurationRequest newInstance(
+ public static SetSubClusterPolicyConfigurationRequest newInstance(
SubClusterPolicyConfiguration policy) {
SetSubClusterPolicyConfigurationRequest request =
Records.newRecord(SetSubClusterPolicyConfigurationRequest.class);
@@ -41,24 +41,6 @@ public abstract class SetSubClusterPolicyConfigurationRequest {
}
/**
- * Get the name of the queue for which we are configuring a policy.
- *
- * @return the name of the queue
- */
- @Public
- @Unstable
- public abstract String getQueue();
-
- /**
- * Sets the name of the queue for which we are configuring a policy.
- *
- * @param queueName the name of the queue
- */
- @Private
- @Unstable
- public abstract void setQueue(String queueName);
-
- /**
* Get the policy configuration assigned to the queue.
*
* @return the policy for the specified queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
index 33c4043..401e984 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.util.Records;
@Private
@Unstable
public abstract class SetSubClusterPolicyConfigurationResponse {
- public SetSubClusterPolicyConfigurationResponse newInstance() {
+ public static SetSubClusterPolicyConfigurationResponse newInstance() {
return Records.newRecord(SetSubClusterPolicyConfigurationResponse.class);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
index bc12acb..2839139 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -29,8 +29,8 @@ import java.nio.ByteBuffer;
/**
* {@link SubClusterPolicyConfiguration} is a class that represents a
- * configuration of a policy. It contains a policy type (resolve to a class
- * name) and its params as an opaque {@link ByteBuffer}.
+ * configuration of a policy. For a single queue, it contains a policy type
+ * (resolve to a class name) and its params as an opaque {@link ByteBuffer}.
*
* Note: by design the params are an opaque ByteBuffer, this allows for enough
* flexibility to evolve the policies without impacting the protocols to/from
@@ -42,16 +42,35 @@ public abstract class SubClusterPolicyConfiguration {
@Private
@Unstable
- public static SubClusterPolicyConfiguration newInstance(String policyType,
- ByteBuffer policyParams) {
+ public static SubClusterPolicyConfiguration newInstance(String queue,
+ String policyType, ByteBuffer policyParams) {
SubClusterPolicyConfiguration policy =
Records.newRecord(SubClusterPolicyConfiguration.class);
+ policy.setQueue(queue);
policy.setType(policyType);
policy.setParams(policyParams);
return policy;
}
/**
+ * Get the name of the queue for which we are configuring a policy.
+ *
+ * @return the name of the queue
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Sets the name of the queue for which we are configuring a policy.
+ *
+ * @param queueName the name of the queue
+ */
+ @Private
+ @Unstable
+ public abstract void setQueue(String queueName);
+
+ /**
* Get the type of the policy. This could be random, round-robin, load-based,
* etc.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
index 865d0c4..585ba81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
@@ -108,6 +108,10 @@ public class GetApplicationHomeSubClusterRequestPBImpl
public ApplicationId getApplicationId() {
GetApplicationHomeSubClusterRequestProtoOrBuilder p =
viaProto ? proto : builder;
+ if (applicationId != null) {
+ return applicationId;
+ }
+
if (!p.hasApplicationId()) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
index 5e29bd5..7b7f89d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -107,23 +107,6 @@ public class SetSubClusterPolicyConfigurationRequestPBImpl
}
@Override
- public String getQueue() {
- SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
- viaProto ? proto : builder;
- return p.getQueue();
- }
-
- @Override
- public void setQueue(String queueName) {
- maybeInitBuilder();
- if (queueName == null) {
- builder.clearQueue();
- return;
- }
- builder.setQueue(queueName);
- }
-
- @Override
public SubClusterPolicyConfiguration getPolicyConfiguration() {
SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
viaProto ? proto : builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
index fe9d9db..305a8d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
@@ -87,6 +87,23 @@ public class SubClusterPolicyConfigurationPBImpl
}
@Override
+ public String getQueue() {
+ SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearType();
+ return;
+ }
+ builder.setQueue(queueName);
+
+ }
+
+ @Override
public String getType() {
SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
return p.getType();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 3f1cee9..11f786f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -136,8 +136,9 @@ message DeleteApplicationHomeSubClusterResponseProto {
}
message SubClusterPolicyConfigurationProto {
- optional string type = 1;
- optional bytes params = 2;
+ optional string queue = 1;
+ optional string type = 2;
+ optional bytes params = 3;
}
message GetSubClusterPolicyConfigurationRequestProto {
@@ -149,8 +150,7 @@ message GetSubClusterPolicyConfigurationResponseProto {
}
message SetSubClusterPolicyConfigurationRequestProto {
- optional string queue = 1;
- optional SubClusterPolicyConfigurationProto policy_configuration = 2;
+ optional SubClusterPolicyConfigurationProto policy_configuration = 1;
}
message SetSubClusterPolicyConfigurationResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index c76a485..165dd78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -18,18 +18,39 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
import org.apache.hadoop.yarn.util.MonotonicClock;
import org.junit.After;
import org.junit.Assert;
@@ -42,20 +63,21 @@ import org.junit.Test;
public abstract class FederationStateStoreBaseTest {
private static final MonotonicClock CLOCK = new MonotonicClock();
+ private FederationStateStore stateStore = createStateStore();
- private FederationMembershipStateStore stateStore;
+ protected abstract FederationStateStore createStateStore();
@Before
- public void before() throws IOException {
- stateStore = getCleanStateStore();
+ public void before() throws IOException, YarnException {
+ stateStore.init(new Configuration());
}
@After
- public void after() {
- stateStore = null;
+ public void after() throws Exception {
+ stateStore.close();
}
- protected abstract FederationMembershipStateStore getCleanStateStore();
+ // Test FederationMembershipStateStore
@Test
public void testRegisterSubCluster() throws Exception {
@@ -72,10 +94,7 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testDeregisterSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
-
- stateStore.registerSubCluster(
- SubClusterRegisterRequest.newInstance(subClusterInfo));
+ registerSubCluster(subClusterId);
SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
.newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
@@ -105,9 +124,7 @@ public abstract class FederationStateStoreBaseTest {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
-
- stateStore.registerSubCluster(
- SubClusterRegisterRequest.newInstance(subClusterInfo));
+ registerSubCluster(subClusterId);
GetSubClusterInfoRequest request =
GetSubClusterInfoRequest.newInstance(subClusterId);
@@ -167,10 +184,7 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testSubClusterHeartbeat() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
-
- stateStore.registerSubCluster(
- SubClusterRegisterRequest.newInstance(subClusterInfo));
+ registerSubCluster(subClusterId);
SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
.newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
@@ -196,6 +210,271 @@ public abstract class FederationStateStoreBaseTest {
}
}
+ // Test FederationApplicationHomeSubClusterStore
+
+ @Test
+ public void testAddApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc);
+ AddApplicationHomeSubClusterResponse response =
+ stateStore.addApplicationHomeSubClusterMap(request);
+
+ Assert.assertNotNull(response);
+ Assert.assertEquals(subClusterId, queryApplicationHomeSC(appId));
+
+ }
+
+ @Test
+ public void testAddApplicationHomeSubClusterMapAppAlreadyExists()
+ throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ addApplicationHomeSC(appId, subClusterId1);
+
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ ApplicationHomeSubCluster ahsc2 =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
+
+ try {
+ stateStore.addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc2));
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " already exists"));
+ }
+
+ Assert.assertEquals(subClusterId1, queryApplicationHomeSC(appId));
+
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ addApplicationHomeSC(appId, subClusterId);
+
+ DeleteApplicationHomeSubClusterRequest delRequest =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+
+ DeleteApplicationHomeSubClusterResponse response =
+ stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+
+ Assert.assertNotNull(response);
+ try {
+ queryApplicationHomeSC(appId);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId + " does not exist"));
+ }
+
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterMapUnknownApp()
+ throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ DeleteApplicationHomeSubClusterRequest delRequest =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+
+ try {
+ stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ addApplicationHomeSC(appId, subClusterId);
+
+ GetApplicationHomeSubClusterRequest getRequest =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ GetApplicationHomeSubClusterResponse result =
+ stateStore.getApplicationHomeSubClusterMap(getRequest);
+
+ Assert.assertEquals(appId,
+ result.getApplicationHomeSubCluster().getApplicationId());
+ Assert.assertEquals(subClusterId,
+ result.getApplicationHomeSubCluster().getHomeSubCluster());
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterMapUnknownApp() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ try {
+ stateStore.getApplicationHomeSubClusterMap(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetApplicationsHomeSubClusterMap() throws Exception {
+ ApplicationId appId1 = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ ApplicationHomeSubCluster ahsc1 =
+ ApplicationHomeSubCluster.newInstance(appId1, subClusterId1);
+
+ ApplicationId appId2 = ApplicationId.newInstance(1, 2);
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ ApplicationHomeSubCluster ahsc2 =
+ ApplicationHomeSubCluster.newInstance(appId2, subClusterId2);
+
+ addApplicationHomeSC(appId1, subClusterId1);
+ addApplicationHomeSC(appId2, subClusterId2);
+
+ GetApplicationsHomeSubClusterRequest getRequest =
+ GetApplicationsHomeSubClusterRequest.newInstance();
+
+ GetApplicationsHomeSubClusterResponse result =
+ stateStore.getApplicationsHomeSubClusterMap(getRequest);
+
+ Assert.assertEquals(2, result.getAppsHomeSubClusters().size());
+ Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc1));
+ Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc2));
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ addApplicationHomeSC(appId, subClusterId1);
+
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ ApplicationHomeSubCluster ahscUpdate =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
+
+ UpdateApplicationHomeSubClusterRequest updateRequest =
+ UpdateApplicationHomeSubClusterRequest.newInstance(ahscUpdate);
+
+ UpdateApplicationHomeSubClusterResponse response =
+ stateStore.updateApplicationHomeSubClusterMap(updateRequest);
+
+ Assert.assertNotNull(response);
+
+ Assert.assertEquals(subClusterId2, queryApplicationHomeSC(appId));
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterMapUnknownApp()
+ throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId1);
+
+ UpdateApplicationHomeSubClusterRequest updateRequest =
+ UpdateApplicationHomeSubClusterRequest.newInstance(ahsc);
+
+ try {
+ stateStore.updateApplicationHomeSubClusterMap((updateRequest));
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " does not exist"));
+ }
+ }
+
+ // Test FederationPolicyStore
+
+ @Test
+ public void testSetPolicyConfiguration() throws Exception {
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf("Queue", "PolicyType"));
+
+ SetSubClusterPolicyConfigurationResponse result =
+ stateStore.setPolicyConfiguration(request);
+
+ Assert.assertNotNull(result);
+ Assert.assertEquals(createSCPolicyConf("Queue", "PolicyType"),
+ queryPolicy("Queue"));
+
+ }
+
+ @Test
+ public void testSetPolicyConfigurationUpdateExisting() throws Exception {
+ setPolicyConf("Queue", "PolicyType1");
+
+ SetSubClusterPolicyConfigurationRequest request2 =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf("Queue", "PolicyType2"));
+ SetSubClusterPolicyConfigurationResponse result =
+ stateStore.setPolicyConfiguration(request2);
+
+ Assert.assertNotNull(result);
+ Assert.assertEquals(createSCPolicyConf("Queue", "PolicyType2"),
+ queryPolicy("Queue"));
+ }
+
+ @Test
+ public void testGetPolicyConfiguration() throws Exception {
+ setPolicyConf("Queue", "PolicyType");
+
+ GetSubClusterPolicyConfigurationRequest getRequest =
+ GetSubClusterPolicyConfigurationRequest.newInstance("Queue");
+ GetSubClusterPolicyConfigurationResponse result =
+ stateStore.getPolicyConfiguration(getRequest);
+
+ Assert.assertNotNull(result);
+ Assert.assertEquals(createSCPolicyConf("Queue", "PolicyType"),
+ result.getPolicyConfiguration());
+
+ }
+
+ @Test
+ public void testGetPolicyConfigurationUnknownQueue() throws Exception {
+
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance("Queue");
+ try {
+ stateStore.getPolicyConfiguration(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Policy for queue Queue does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetPoliciesConfigurations() throws Exception {
+ setPolicyConf("Queue1", "PolicyType1");
+ setPolicyConf("Queue2", "PolicyType2");
+
+ GetSubClusterPoliciesConfigurationsResponse response =
+ stateStore.getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest.newInstance());
+
+ Assert.assertNotNull(response);
+ Assert.assertNotNull(response.getPoliciesConfigs());
+
+ Assert.assertEquals(2, response.getPoliciesConfigs().size());
+
+ Assert.assertTrue(response.getPoliciesConfigs()
+ .contains(createSCPolicyConf("Queue1", "PolicyType1")));
+ Assert.assertTrue(response.getPoliciesConfigs()
+ .contains(createSCPolicyConf("Queue2", "PolicyType2")));
+ }
+
+ // Convenience methods
+
private SubClusterInfo createSubClusterInfo(SubClusterId subClusterId) {
String amRMAddress = "1.2.3.4:1";
@@ -208,6 +487,37 @@ public abstract class FederationStateStoreBaseTest {
CLOCK.getTime(), "cabability");
}
+ private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
+ String policyType) {
+ return SubClusterPolicyConfiguration.newInstance(queueName, policyType,
+ ByteBuffer.allocate(1));
+ }
+
+ private void addApplicationHomeSC(ApplicationId appId,
+ SubClusterId subClusterId) throws YarnException {
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc);
+ stateStore.addApplicationHomeSubClusterMap(request);
+ }
+
+ private void setPolicyConf(String queue, String policyType)
+ throws YarnException {
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf(queue, policyType));
+ stateStore.setPolicyConfiguration(request);
+ }
+
+ private void registerSubCluster(SubClusterId subClusterId)
+ throws YarnException {
+
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ }
+
private SubClusterInfo querySubClusterInfo(SubClusterId subClusterId)
throws YarnException {
GetSubClusterInfoRequest request =
@@ -215,4 +525,25 @@ public abstract class FederationStateStoreBaseTest {
return stateStore.getSubCluster(request).getSubClusterInfo();
}
+ private SubClusterId queryApplicationHomeSC(ApplicationId appId)
+ throws YarnException {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ GetApplicationHomeSubClusterResponse response =
+ stateStore.getApplicationHomeSubClusterMap(request);
+
+ return response.getApplicationHomeSubCluster().getHomeSubCluster();
+ }
+
+ private SubClusterPolicyConfiguration queryPolicy(String queue)
+ throws YarnException {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue);
+
+ GetSubClusterPolicyConfigurationResponse result =
+ stateStore.getPolicyConfiguration(request);
+ return result.getPolicyConfiguration();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed2c6c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
index 9396eda..74404c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
-import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
/**
* Unit tests for MemoryFederationStateStore.
@@ -26,7 +26,7 @@ public class TestMemoryFederationStateStore
extends FederationStateStoreBaseTest {
@Override
- protected FederationMembershipStateStore getCleanStateStore() {
+ protected FederationStateStore createStateStore() {
return new MemoryFederationStateStore();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-3671. Integrate Federation
services with ResourceManager. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-3671. Integrate Federation services with ResourceManager. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ef27f1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ef27f1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ef27f1b
Branch: refs/heads/YARN-2915
Commit: 5ef27f1b0f2211a222b98bbf2c8d7212c76bb1fc
Parents: e637a07
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 30 12:20:52 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 11 +-
.../yarn/conf/TestYarnConfigurationFields.java | 4 +-
.../failover/FederationProxyProviderUtil.java | 2 +-
.../FederationRMFailoverProxyProvider.java | 4 +-
...ationMembershipStateStoreInputValidator.java | 7 +-
.../TestFederationStateStoreInputValidator.java | 10 +-
.../server/resourcemanager/ResourceManager.java | 26 ++
.../FederationStateStoreHeartbeat.java | 108 +++++++
.../federation/FederationStateStoreService.java | 304 +++++++++++++++++++
.../federation/package-info.java | 17 ++
.../webapp/dao/ClusterMetricsInfo.java | 5 +-
.../TestFederationRMStateStoreService.java | 170 +++++++++++
12 files changed, 648 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a2c42fd..f3062e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2494,9 +2494,6 @@ public class YarnConfiguration extends Configuration {
FEDERATION_PREFIX + "failover.enabled";
public static final boolean DEFAULT_FEDERATION_FAILOVER_ENABLED = true;
- public static final String FEDERATION_SUBCLUSTER_ID =
- FEDERATION_PREFIX + "sub-cluster.id";
-
public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
FEDERATION_PREFIX + "state-store.class";
@@ -2509,6 +2506,14 @@ public class YarnConfiguration extends Configuration {
// 5 minutes
public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+ public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
+ FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
+
+ // 5 minutes
+ public static final int
+ DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
+ 5 * 60;
+
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index c4d8f38..5e0876f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -72,9 +72,9 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
configurationPropsToSkipCompare
- .add(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
- configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
index a986008..18f1338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
@@ -134,7 +134,7 @@ public final class FederationProxyProviderUtil {
// are based out of conf
private static void updateConf(Configuration conf,
SubClusterId subClusterId) {
- conf.set(YarnConfiguration.FEDERATION_SUBCLUSTER_ID, subClusterId.getId());
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
// In a Federation setting, we will connect to not just the local cluster RM
// but also multiple external RMs. The membership information of all the RMs
// that are currently
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index 90a9239..0ffab0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -74,8 +74,8 @@ public class FederationRMFailoverProxyProvider<T>
this.protocol = proto;
this.rmProxy.checkAllowedProtocols(this.protocol);
String clusterId =
- configuration.get(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
- Preconditions.checkNotNull(clusterId, "Missing Federation SubClusterId");
+ configuration.get(YarnConfiguration.RM_CLUSTER_ID);
+ Preconditions.checkNotNull(clusterId, "Missing RM ClusterId");
this.subClusterId = SubClusterId.newInstance(clusterId);
this.facade = facade.getInstance();
if (configuration instanceof YarnConfiguration) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
index b587ee5..ff9d8e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
@@ -159,7 +159,10 @@ public final class FederationMembershipStateStoreInputValidator {
}
/**
- * Validate if the SubCluster Info are present or not.
+ * Validate if all the required fields on {@link SubClusterInfo} are present
+ * or not. {@code Capability} will be empty as the corresponding
+ * {@code ResourceManager} is in the process of initialization during
+ * registration.
*
* @param subClusterInfo the information of the subcluster to be verified
* @throws FederationStateStoreInvalidInputException if the SubCluster Info
@@ -194,8 +197,6 @@ public final class FederationMembershipStateStoreInputValidator {
// validate subcluster state
checkSubClusterState(subClusterInfo.getState());
- // validate subcluster capability
- checkCapability(subClusterInfo.getCapability());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
index 13175ae..b95f17a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
@@ -242,11 +242,8 @@ public class TestFederationStateStoreInputValidator {
SubClusterRegisterRequest.newInstance(subClusterInfo);
FederationMembershipStateStoreInputValidator
.validateSubClusterRegisterRequest(request);
- Assert.fail();
} catch (FederationStateStoreInvalidInputException e) {
- LOG.info(e.getMessage());
- Assert.assertTrue(
- e.getMessage().startsWith("Invalid capability information."));
+ Assert.fail(e.getMessage());
}
// Execution with Empty Capability
@@ -260,11 +257,8 @@ public class TestFederationStateStoreInputValidator {
SubClusterRegisterRequest.newInstance(subClusterInfo);
FederationMembershipStateStoreInputValidator
.validateSubClusterRegisterRequest(request);
- Assert.fail();
} catch (FederationStateStoreInvalidInputException e) {
- LOG.info(e.getMessage());
- Assert.assertTrue(
- e.getMessage().startsWith("Invalid capability information."));
+ Assert.fail(e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 58e4077..e4bd64e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.federation.FederationStateStoreService;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV1Publisher;
@@ -188,6 +189,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected RMAppManager rmAppManager;
protected ApplicationACLsManager applicationACLsManager;
protected QueueACLsManager queueACLsManager;
+ private FederationStateStoreService federationStateStoreService;
private WebApp webApp;
private AppReportFetcher fetcher = null;
protected ResourceTrackerService resourceTracker;
@@ -504,6 +506,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
return new RMTimelineCollectorManager(rmContext);
}
+ private FederationStateStoreService createFederationStateStoreService() {
+ return new FederationStateStoreService(rmContext);
+ }
+
protected SystemMetricsPublisher createSystemMetricsPublisher() {
SystemMetricsPublisher publisher;
if (YarnConfiguration.timelineServiceEnabled(conf) &&
@@ -733,6 +739,20 @@ public class ResourceManager extends CompositeService implements Recoverable {
delegationTokenRenewer.setRMContext(rmContext);
}
+ if(HAUtil.isFederationEnabled(conf)) {
+ String cId = YarnConfiguration.getClusterId(conf);
+ if (cId.isEmpty()) {
+ String errMsg =
+ "Cannot initialize RM as Federation is enabled"
+ + " but cluster id is not configured.";
+ LOG.error(errMsg);
+ throw new YarnRuntimeException(errMsg);
+ }
+ federationStateStoreService = createFederationStateStoreService();
+ addIfService(federationStateStoreService);
+ LOG.info("Initialized Federation membership.");
+ }
+
new RMNMInfo(rmContext, scheduler);
super.serviceInit(conf);
@@ -1397,6 +1417,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
@Private
+ @VisibleForTesting
+ public FederationStateStoreService getFederationStateStoreService() {
+ return this.federationStateStoreService;
+ }
+
+ @Private
WebApp getWebapp() {
return this.webApp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java
new file mode 100644
index 0000000..a4618a2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import java.io.StringWriter;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+
+/**
+ * Periodic heart beat from a <code>ResourceManager</code> participating in
+ * federation to indicate liveliness. The heart beat publishes the current
+ * capabilities as represented by {@link ClusterMetricsInfo} of the sub cluster.
+ *
+ */
+public class FederationStateStoreHeartbeat implements Runnable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreHeartbeat.class);
+
+ private SubClusterId subClusterId;
+ private FederationStateStore stateStoreService;
+
+ private final ResourceScheduler rs;
+
+ private StringWriter currentClusterState;
+ private JSONJAXBContext jc;
+ private JSONMarshaller marshaller;
+ private String capability;
+
+ public FederationStateStoreHeartbeat(SubClusterId subClusterId,
+ FederationStateStore stateStoreClient, ResourceScheduler scheduler) {
+ this.stateStoreService = stateStoreClient;
+ this.subClusterId = subClusterId;
+ this.rs = scheduler;
+ // Initialize the JAXB Marshaller
+ this.currentClusterState = new StringWriter();
+ try {
+ this.jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
+ ClusterMetricsInfo.class);
+ marshaller = jc.createJSONMarshaller();
+ } catch (JAXBException e) {
+ LOG.warn("Exception while trying to initialize JAXB context.", e);
+ }
+ LOG.info("Initialized Federation membership for cluster with timestamp: "
+ + ResourceManager.getClusterTimeStamp());
+ }
+
+ /**
+ * Get the current cluster state as a JSON string representation of the
+ * {@link ClusterMetricsInfo}.
+ */
+ private void updateClusterState() {
+ try {
+ // get the current state
+ currentClusterState.getBuffer().setLength(0);
+ ClusterMetricsInfo clusterMetricsInfo = new ClusterMetricsInfo(rs);
+ marshaller.marshallToJSON(clusterMetricsInfo, currentClusterState);
+ capability = currentClusterState.toString();
+ } catch (Exception e) {
+ LOG.warn("Exception while trying to generate cluster state,"
+ + " so reverting to last know state.", e);
+ }
+ }
+
+ @Override
+ public synchronized void run() {
+ try {
+ updateClusterState();
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, capability);
+ stateStoreService.subClusterHeartbeat(request);
+ LOG.debug("Sending the heartbeat with capability: {}", capability);
+ } catch (Exception e) {
+ LOG.warn("Exception when trying to heartbeat: ", e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
new file mode 100644
index 0000000..9a01d7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Implements {@link FederationStateStore} and provides a service for
+ * participating in the federation membership.
+ */
+public class FederationStateStoreService extends AbstractService
+ implements FederationStateStore {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreService.class);
+
+ private Configuration config;
+ private ScheduledExecutorService scheduledExecutorService;
+ private FederationStateStoreHeartbeat stateStoreHeartbeat;
+ private FederationStateStore stateStoreClient = null;
+ private SubClusterId subClusterId;
+ private long heartbeatInterval;
+ private RMContext rmContext;
+
+ public FederationStateStoreService(RMContext rmContext) {
+ super(FederationStateStoreService.class.getName());
+ LOG.info("FederationStateStoreService initialized");
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ this.config = conf;
+
+ RetryPolicy retryPolicy =
+ FederationStateStoreFacade.createRetryPolicy(conf);
+
+ this.stateStoreClient =
+ (FederationStateStore) FederationStateStoreFacade.createRetryInstance(
+ conf, YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS,
+ FederationStateStore.class, retryPolicy);
+ this.stateStoreClient.init(conf);
+ LOG.info("Initialized state store client class");
+
+ this.subClusterId =
+ SubClusterId.newInstance(YarnConfiguration.getClusterId(conf));
+
+ heartbeatInterval = conf.getLong(
+ YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+ if (heartbeatInterval <= 0) {
+ heartbeatInterval =
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS;
+ }
+ LOG.info("Initialized federation membership service.");
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ registerAndInitializeHeartbeat();
+
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ Exception ex = null;
+ try {
+ if (this.scheduledExecutorService != null
+ && !this.scheduledExecutorService.isShutdown()) {
+ this.scheduledExecutorService.shutdown();
+ LOG.info("Stopped federation membership heartbeat");
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to shutdown ScheduledExecutorService", e);
+ ex = e;
+ }
+
+ if (this.stateStoreClient != null) {
+ try {
+ deregisterSubCluster(SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED));
+ } finally {
+ this.stateStoreClient.close();
+ }
+ }
+
+ if (ex != null) {
+ throw ex;
+ }
+ }
+
+ // Return a client accessible string representation of the service address.
+ private String getServiceAddress(InetSocketAddress address) {
+ InetSocketAddress socketAddress = NetUtils.getConnectAddress(address);
+ return socketAddress.getAddress().getHostAddress() + ":"
+ + socketAddress.getPort();
+ }
+
+ private void registerAndInitializeHeartbeat() {
+ String clientRMAddress =
+ getServiceAddress(rmContext.getClientRMService().getBindAddress());
+ String amRMAddress = getServiceAddress(
+ rmContext.getApplicationMasterService().getBindAddress());
+ String rmAdminAddress = getServiceAddress(
+ config.getSocketAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
+ String webAppAddress =
+ WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(config);
+
+ SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMAddress, clientRMAddress, rmAdminAddress, webAppAddress,
+ SubClusterState.SC_NEW, ResourceManager.getClusterTimeStamp(), "");
+ try {
+ registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo));
+ LOG.info("Successfully registered for federation subcluster: {}",
+ subClusterInfo);
+ } catch (Exception e) {
+ throw new YarnRuntimeException(
+ "Failed to register Federation membership with the StateStore", e);
+ }
+ stateStoreHeartbeat = new FederationStateStoreHeartbeat(subClusterId,
+ stateStoreClient, rmContext.getScheduler());
+ scheduledExecutorService =
+ HadoopExecutors.newSingleThreadScheduledExecutor();
+ scheduledExecutorService.scheduleWithFixedDelay(stateStoreHeartbeat,
+ heartbeatInterval, heartbeatInterval, TimeUnit.SECONDS);
+ LOG.info("Started federation membership heartbeat with interval: {}",
+ heartbeatInterval);
+ }
+
+ @VisibleForTesting
+ public FederationStateStore getStateStoreClient() {
+ return stateStoreClient;
+ }
+
+ @VisibleForTesting
+ public FederationStateStoreHeartbeat getStateStoreHeartbeatThread() {
+ return stateStoreHeartbeat;
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return stateStoreClient.getCurrentVersion();
+ }
+
+ @Override
+ public Version loadVersion() {
+ return stateStoreClient.getCurrentVersion();
+ }
+
+ @Override
+ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ return stateStoreClient.getPolicyConfiguration(request);
+ }
+
+ @Override
+ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ return stateStoreClient.setPolicyConfiguration(request);
+ }
+
+ @Override
+ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException {
+ return stateStoreClient.getPoliciesConfigurations(request);
+ }
+
+ @Override
+ public SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest registerSubClusterRequest)
+ throws YarnException {
+ return stateStoreClient.registerSubCluster(registerSubClusterRequest);
+ }
+
+ @Override
+ public SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest subClusterDeregisterRequest)
+ throws YarnException {
+ return stateStoreClient.deregisterSubCluster(subClusterDeregisterRequest);
+ }
+
+ @Override
+ public SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest subClusterHeartbeatRequest)
+ throws YarnException {
+ return stateStoreClient.subClusterHeartbeat(subClusterHeartbeatRequest);
+ }
+
+ @Override
+ public GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest subClusterRequest) throws YarnException {
+ return stateStoreClient.getSubCluster(subClusterRequest);
+ }
+
+ @Override
+ public GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest subClustersRequest) throws YarnException {
+ return stateStoreClient.getSubClusters(subClustersRequest);
+ }
+
+ @Override
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.addApplicationHomeSubCluster(request);
+ }
+
+ @Override
+ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
+ UpdateApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.updateApplicationHomeSubCluster(request);
+ }
+
+ @Override
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
+ GetApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.getApplicationHomeSubCluster(request);
+ }
+
+ @Override
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.getApplicationsHomeSubCluster(request);
+ }
+
+ @Override
+ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
+ DeleteApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.deleteApplicationHomeSubCluster(request);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java
new file mode 100644
index 0000000..47c7c65
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index f083b05..dc42eb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -65,7 +65,10 @@ public class ClusterMetricsInfo {
} // JAXB needs this
public ClusterMetricsInfo(final ResourceManager rm) {
- ResourceScheduler rs = rm.getResourceScheduler();
+ this(rm.getResourceScheduler());
+ }
+
+ public ClusterMetricsInfo(final ResourceScheduler rs) {
QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ef27f1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
new file mode 100644
index 0000000..30f69b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+
+/**
+ * Unit tests for FederationStateStoreService.
+ */
+public class TestFederationRMStateStoreService {
+
+ private final HAServiceProtocol.StateChangeRequestInfo requestInfo =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ private final SubClusterId subClusterId = SubClusterId.newInstance("SC-1");
+ private final GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private long lastHearbeatTS = 0;
+ private JSONJAXBContext jc;
+ private JSONUnmarshaller unmarshaller;
+
+ @Before
+ public void setUp() throws IOException, YarnException, JAXBException {
+ conf = new YarnConfiguration();
+ jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
+ ClusterMetricsInfo.class);
+ unmarshaller = jc.createJSONUnmarshaller();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ unmarshaller = null;
+ jc = null;
+ }
+
+ @Test
+ public void testFederationStateStoreService() throws Exception {
+ conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
+ final MockRM rm = new MockRM(conf);
+
+ // Initially there should be no entry for the sub-cluster
+ rm.init(conf);
+ stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+ try {
+ stateStore.getSubCluster(request);
+ Assert.fail("There should be no entry for the sub-cluster.");
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().endsWith("does not exist"));
+ }
+
+ // Validate if sub-cluster is registered
+ rm.start();
+ String capability = checkSubClusterInfo(SubClusterState.SC_NEW);
+ Assert.assertTrue(capability.isEmpty());
+
+ // Heartbeat to see if sub-cluster transitions to running
+ FederationStateStoreHeartbeat storeHeartbeat =
+ rm.getFederationStateStoreService().getStateStoreHeartbeatThread();
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 0);
+
+ // heartbeat again after adding a node.
+ rm.registerNode("127.0.0.1:1234", 4 * 1024);
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 1);
+
+ // Validate sub-cluster deregistration
+ rm.getFederationStateStoreService()
+ .deregisterSubCluster(SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED));
+ checkSubClusterInfo(SubClusterState.SC_UNREGISTERED);
+
+ // check after failover
+ explicitFailover(rm);
+
+ capability = checkSubClusterInfo(SubClusterState.SC_NEW);
+ Assert.assertTrue(capability.isEmpty());
+
+ // Heartbeat to see if sub-cluster transitions to running
+ storeHeartbeat =
+ rm.getFederationStateStoreService().getStateStoreHeartbeatThread();
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 0);
+
+ // heartbeat again after adding a node.
+ rm.registerNode("127.0.0.1:1234", 4 * 1024);
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 1);
+
+ rm.stop();
+ }
+
+ private void explicitFailover(MockRM rm) throws IOException {
+ rm.getAdminService().transitionToStandby(requestInfo);
+ Assert.assertTrue(rm.getRMContext()
+ .getHAServiceState() == HAServiceProtocol.HAServiceState.STANDBY);
+ rm.getAdminService().transitionToActive(requestInfo);
+ Assert.assertTrue(rm.getRMContext()
+ .getHAServiceState() == HAServiceProtocol.HAServiceState.ACTIVE);
+ lastHearbeatTS = 0;
+ stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+ }
+
+ private void checkClusterMetricsInfo(String capability, int numNodes)
+ throws JAXBException {
+ ClusterMetricsInfo clusterMetricsInfo = unmarshaller.unmarshalFromJSON(
+ new StringReader(capability), ClusterMetricsInfo.class);
+ Assert.assertEquals(numNodes, clusterMetricsInfo.getTotalNodes());
+ }
+
+ private String checkSubClusterInfo(SubClusterState state)
+ throws YarnException {
+ Assert.assertNotNull(stateStore.getSubCluster(request));
+ SubClusterInfo response =
+ stateStore.getSubCluster(request).getSubClusterInfo();
+ Assert.assertEquals(state, response.getState());
+ Assert.assertTrue(response.getLastHeartBeat() >= lastHearbeatTS);
+ lastHearbeatTS = response.getLastHeartBeat();
+ return response.getCapability();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5634. Simplify
initialization/use of RouterPolicy via a RouterPolicyFacade. (Carlo Curino
via Subru).
Posted by su...@apache.org.
YARN-5634. Simplify initialization/use of RouterPolicy via a RouterPolicyFacade. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbbeec41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbbeec41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbbeec41
Branch: refs/heads/YARN-2915
Commit: fbbeec418a898e35d24672ec7a1d75162e34b901
Parents: 112b99b
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Nov 16 19:39:25 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 9 +
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../yarn/conf/TestYarnConfigurationFields.java | 12 +
...ionPolicyInitializationContextValidator.java | 2 +-
.../PriorityBroadcastPolicyManager.java | 66 +++++
.../federation/policies/RouterPolicyFacade.java | 266 +++++++++++++++++++
.../policies/dao/WeightedPolicyInfo.java | 6 +-
.../utils/FederationStateStoreFacade.java | 16 +-
.../TestPriorityBroadcastPolicyManager.java | 72 +++++
.../policies/TestRouterPolicyFacade.java | 220 +++++++++++++++
.../utils/FederationStateStoreTestUtil.java | 22 +-
11 files changed, 693 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index bbd03a9..ee51094 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -310,6 +310,15 @@
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade"/>
+ <Or>
+ <Field name="globalConfMap"/>
+ <Field name="globalPolicyMap"/>
+ </Or>
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+ </Match>
+
<!-- Don't care if putIfAbsent value is ignored -->
<Match>
<Package name="org.apache.hadoop.yarn.factories.impl.pb" />
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 58b2f44..aca1177 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2520,6 +2520,19 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
+ public static final String DEFAULT_FEDERATION_POLICY_KEY = "*";
+
+ public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX
+ + "policy-manager";
+
+ public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
+ + ".hadoop.yarn.server.federation.policies.UniformBroadcastPolicyManager";
+
+ public static final String FEDERATION_POLICY_MANAGER_PARAMS =
+ FEDERATION_PREFIX + "policy-manager-params";
+
+ public static final String DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS = "";
+
////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3f3a06c..6e33c0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -78,6 +78,18 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_EPOCH);
+ // Federation policies configs to be ignored
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_POLICY_MANAGER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
configurationPropsToSkipCompare.add(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
index 1b83bbc..3c44e7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
@@ -57,7 +57,7 @@ public final class FederationPolicyInitializationContextValidator {
if (policyContext.getSubClusterPolicyConfiguration() == null) {
throw new FederationPolicyInitializationException(
- "The FederationSubclusterResolver provided is null. Cannot "
+ "The SubClusterPolicyConfiguration provided is null. Cannot "
+ "reinitalize successfully.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..ebdcf42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link PriorityRouterPolicy} for the router and a
+ * {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class PriorityBroadcastPolicyManager extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public PriorityBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = PriorityRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration.newInstance(getQueue(),
+ this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
new file mode 100644
index 0000000..a3fd15a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This class provides a facade to the policy subsystem, and handles the
+ * lifecycle of policies (e.g., refresh from remote, default behaviors etc.).
+ */
+public class RouterPolicyFacade {
+
+ private static final Log LOG =
+ LogFactory.getLog(LocalityMulticastAMRMProxyPolicy.class);
+
+ private final SubClusterResolver subClusterResolver;
+ private final FederationStateStoreFacade federationFacade;
+ private Map<String, SubClusterPolicyConfiguration> globalConfMap;
+
+ @VisibleForTesting
+ Map<String, FederationRouterPolicy> globalPolicyMap;
+
+ public RouterPolicyFacade(YarnConfiguration conf,
+ FederationStateStoreFacade facade, SubClusterResolver resolver,
+ SubClusterId homeSubcluster)
+ throws FederationPolicyInitializationException {
+
+ this.federationFacade = facade;
+ this.subClusterResolver = resolver;
+ this.globalConfMap = new ConcurrentHashMap<>();
+ this.globalPolicyMap = new ConcurrentHashMap<>();
+
+ // load default behavior from store if possible
+ String defaulKey = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+ SubClusterPolicyConfiguration configuration = null;
+ try {
+ configuration = federationFacade.getPolicyConfiguration(defaulKey);
+ } catch (YarnException e) {
+ LOG.warn("No fallback behavior defined in store, defaulting to XML "
+ + "configuration fallback behavior.");
+ }
+
+ // or from XML conf otherwise.
+ if (configuration == null) {
+ String defaultFederationPolicyManager =
+ conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER,
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER);
+ String defaultPolicyParamString =
+ conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS,
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+ ByteBuffer defaultPolicyParam = ByteBuffer
+ .wrap(defaultPolicyParamString.getBytes(StandardCharsets.UTF_8));
+
+ configuration = SubClusterPolicyConfiguration.newInstance(defaulKey,
+ defaultFederationPolicyManager, defaultPolicyParam);
+ }
+
+ // construct the required policy manager
+ FederationPolicyInitializationContext fallbackContext =
+ new FederationPolicyInitializationContext(configuration,
+ subClusterResolver, federationFacade, homeSubcluster);
+ FederationPolicyManager fallbackPolicyManager =
+ instantiatePolicyManager(configuration.getType());
+ fallbackPolicyManager.setQueue(defaulKey);
+
+ // add to the cache the fallback behavior
+ globalConfMap.put(defaulKey,
+ fallbackContext.getSubClusterPolicyConfiguration());
+ globalPolicyMap.put(defaulKey,
+ fallbackPolicyManager.getRouterPolicy(fallbackContext, null));
+
+ }
+
+ /**
+ * This method provides a wrapper of all policy functionalities for routing .
+ * Internally it manages configuration changes, and policy init/reinit.
+ *
+ * @param appSubmissionContext the application to route.
+ *
+ * @return the id of the subcluster that will be the "home" for this
+ * application.
+ *
+ * @throws YarnException if there are issues initializing policies, or no
+ * valid sub-cluster id could be found for this app.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+
+ // the maps are concurrent, but we need to protect from reset()
+ // reinitialization mid-execution by creating a new reference local to this
+ // method.
+ Map<String, SubClusterPolicyConfiguration> cachedConfs = globalConfMap;
+ Map<String, FederationRouterPolicy> policyMap = globalPolicyMap;
+
+ if (appSubmissionContext == null) {
+ throw new FederationPolicyException(
+ "The ApplicationSubmissionContext " + "cannot be null.");
+ }
+
+ String queue = appSubmissionContext.getQueue();
+
+ // respecting YARN behavior we assume default queue if the queue is not
+ // specified. This also ensures that "null" can be used as a key to get the
+ // default behavior.
+ if (queue == null) {
+ queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
+ }
+
+ // the facade might cache this request, based on its parameterization
+ SubClusterPolicyConfiguration configuration = null;
+
+ try {
+ configuration = federationFacade.getPolicyConfiguration(queue);
+ } catch (YarnException e) {
+ LOG.debug(e);
+ }
+
+ // If there is no policy configured for this queue, fallback to the baseline
+ // policy that is configured either in the store or via XML config (and
+ // cached)
+ if (configuration == null) {
+ try {
+ LOG.warn("There is no policies configured for queue: " + queue + " we"
+ + " fallback to default policy for: "
+ + YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+
+ queue = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+ configuration = federationFacade.getPolicyConfiguration(
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ } catch (YarnException e) {
+ // the fallback is not configure via store, but via XML, using
+ // previously loaded configuration.
+ configuration =
+ cachedConfs.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ }
+ }
+
+ // if the configuration has changed since last loaded, reinit the policy
+ // based on current configuration
+ if (!cachedConfs.containsKey(queue)
+ || !cachedConfs.get(queue).equals(configuration)) {
+ singlePolicyReinit(policyMap, cachedConfs, queue, configuration);
+ }
+
+ FederationRouterPolicy policy = policyMap.get(queue);
+ if (policy == null) {
+ // this should never happen, as the to maps are updated together
+ throw new FederationPolicyException("No FederationRouterPolicy found "
+ + "for queue: " + appSubmissionContext.getQueue() + " (for "
+ + "application: " + appSubmissionContext.getApplicationId() + ") "
+ + "and no default specified.");
+ }
+
+ return policy.getHomeSubcluster(appSubmissionContext);
+ }
+
+ /**
+ * This method reinitializes a policy and loads it in the policyMap.
+ *
+ * @param queue the queue to initialize a policy for.
+ * @param conf the configuration to use for initalization.
+ *
+ * @throws FederationPolicyInitializationException if initialization fails.
+ */
+ private void singlePolicyReinit(Map<String, FederationRouterPolicy> policyMap,
+ Map<String, SubClusterPolicyConfiguration> cachedConfs, String queue,
+ SubClusterPolicyConfiguration conf)
+ throws FederationPolicyInitializationException {
+
+ FederationPolicyInitializationContext context =
+ new FederationPolicyInitializationContext(conf, subClusterResolver,
+ federationFacade, null);
+ String newType = context.getSubClusterPolicyConfiguration().getType();
+ FederationRouterPolicy routerPolicy = policyMap.get(queue);
+
+ FederationPolicyManager federationPolicyManager =
+ instantiatePolicyManager(newType);
+ // set queue, reinit policy if required (implementation lazily check
+ // content of conf), and cache it
+ federationPolicyManager.setQueue(queue);
+ routerPolicy =
+ federationPolicyManager.getRouterPolicy(context, routerPolicy);
+
+ // we need the two put to be atomic (across multiple threads invoking
+ // this and reset operations)
+ synchronized (this) {
+ policyMap.put(queue, routerPolicy);
+ cachedConfs.put(queue, conf);
+ }
+ }
+
+ private static FederationPolicyManager instantiatePolicyManager(
+ String newType) throws FederationPolicyInitializationException {
+ FederationPolicyManager federationPolicyManager = null;
+ try {
+ // create policy instance and set queue
+ Class c = Class.forName(newType);
+ federationPolicyManager = (FederationPolicyManager) c.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (InstantiationException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (IllegalAccessException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ return federationPolicyManager;
+ }
+
+ /**
+ * This method flushes all cached configurations and policies. This should be
+ * invoked if the facade remains activity after very large churn of queues in
+ * the system.
+ */
+ public synchronized void reset() {
+
+ // remember the fallBack
+ SubClusterPolicyConfiguration conf =
+ globalConfMap.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ FederationRouterPolicy policy =
+ globalPolicyMap.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+
+ globalConfMap = new ConcurrentHashMap<>();
+ globalPolicyMap = new ConcurrentHashMap<>();
+
+ // add to the cache a fallback with keyword null
+ globalConfMap.put(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY, conf);
+ globalPolicyMap.put(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY,
+ policy);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
index 62eb03b..e7b8afe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.federation.policies.dao;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
@@ -100,7 +100,7 @@ public class WeightedPolicyInfo {
JSONUnmarshaller unmarshaller = jsonjaxbContext.createJSONUnmarshaller();
final byte[] bytes = new byte[bb.remaining()];
bb.get(bytes);
- String params = new String(bytes, Charset.forName("UTF-8"));
+ String params = new String(bytes, StandardCharsets.UTF_8);
WeightedPolicyInfo weightedPolicyInfo = unmarshaller.unmarshalFromJSON(
new StringReader(params), WeightedPolicyInfo.class);
@@ -164,7 +164,7 @@ public class WeightedPolicyInfo {
}
try {
String s = toJSONString();
- return ByteBuffer.wrap(s.getBytes(Charset.forName("UTF-8")));
+ return ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8));
} catch (JAXBException j) {
throw new FederationPolicyInitializationException(j);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index 66a0b60..9b794de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoR
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -262,12 +263,17 @@ public final class FederationStateStoreFacade {
if (isCachingEnabled()) {
return getPoliciesConfigurations().get(queue);
} else {
- return stateStore
- .getPolicyConfiguration(
- GetSubClusterPolicyConfigurationRequest.newInstance(queue))
- .getPolicyConfiguration();
- }
+ GetSubClusterPolicyConfigurationResponse response =
+ stateStore.getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue));
+ if (response == null) {
+ throw new YarnException("The stateStore returned a null for "
+ + "GetSubClusterPolicyConfigurationResponse for queue " + queue);
+ } else {
+ return response.getPolicyConfiguration();
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..5e5bc83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test of {@link PriorityBroadcastPolicyManager}.
+ */
+public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new PriorityBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo);
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = PriorityBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = PriorityRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+
+ // check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(
+ ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(),
+ policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
new file mode 100644
index 0000000..4975a9f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test of {@link RouterPolicyFacade}.
+ */
+public class TestRouterPolicyFacade {
+
+ private RouterPolicyFacade routerFacade;
+ private List<SubClusterId> subClusterIds;
+ private FederationStateStore store;
+ private String queue1 = "queue1";
+ private String defQueueKey = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+
+ @Before
+ public void setup() throws YarnException {
+
+ // setting up a store and its facade (with caching off)
+ FederationStateStoreFacade fedFacade =
+ FederationStateStoreFacade.getInstance();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, "0");
+ store = new MemoryFederationStateStore();
+ store.init(conf);
+ fedFacade.reinitialize(store, conf);
+
+ FederationStateStoreTestUtil storeTestUtil =
+ new FederationStateStoreTestUtil(store);
+ storeTestUtil.registerSubClusters(10);
+
+ subClusterIds = storeTestUtil.getAllSubClusterIds(true);
+ store.setPolicyConfiguration(SetSubClusterPolicyConfigurationRequest
+ .newInstance(getUniformPolicy(queue1)));
+
+ SubClusterResolver resolver = FederationPoliciesTestUtil.initResolver();
+ routerFacade = new RouterPolicyFacade(new YarnConfiguration(), fedFacade,
+ resolver, subClusterIds.get(0));
+ }
+
+ @Test
+ public void testConfigurationUpdate() throws YarnException {
+
+ // in this test we see what happens when the configuration is changed
+ // between calls. We achieve this by changing what is in the store.
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ when(applicationSubmissionContext.getQueue()).thenReturn(queue1);
+
+ // first call runs using standard UniformRandomRouterPolicy
+ SubClusterId chosen =
+ routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertTrue(routerFacade.globalPolicyMap
+ .get(queue1) instanceof UniformRandomRouterPolicy);
+
+ // then the operator changes how queue1 is routed setting it to
+ // PriorityRouterPolicy with weights favoring the first subcluster in
+ // subClusterIds.
+ store.setPolicyConfiguration(SetSubClusterPolicyConfigurationRequest
+ .newInstance(getPriorityPolicy(queue1)));
+
+ // second call is routed by new policy PriorityRouterPolicy
+ chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(chosen.equals(subClusterIds.get(0)));
+ Assert.assertTrue(routerFacade.globalPolicyMap
+ .get(queue1) instanceof PriorityRouterPolicy);
+ }
+
+ @Test
+ public void testGetHomeSubcluster() throws YarnException {
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ when(applicationSubmissionContext.getQueue()).thenReturn(queue1);
+
+ // the facade only contains the fallback behavior
+ Assert.assertTrue(routerFacade.globalPolicyMap.containsKey(defQueueKey)
+ && routerFacade.globalPolicyMap.size() == 1);
+
+ // when invoked it returns the expected SubClusterId.
+ SubClusterId chosen =
+ routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+
+ // now the caching of policies must have added an entry for this queue
+ Assert.assertTrue(routerFacade.globalPolicyMap.size() == 2);
+
+ // after the facade is used the policyMap contains the expected policy type.
+ Assert.assertTrue(routerFacade.globalPolicyMap
+ .get(queue1) instanceof UniformRandomRouterPolicy);
+
+ // the facade is again empty after reset
+ routerFacade.reset();
+ // the facade only contains the fallback behavior
+ Assert.assertTrue(routerFacade.globalPolicyMap.containsKey(defQueueKey)
+ && routerFacade.globalPolicyMap.size() == 1);
+
+ }
+
+ @Test
+ public void testFallbacks() throws YarnException {
+
+ // this tests the behavior of the system when the queue requested is
+ // not configured (or null) and there is no default policy configured
+ // for DEFAULT_FEDERATION_POLICY_KEY (*). This is our second line of
+ // defense.
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+
+ // The facade answers also for non-initialized policies (using the
+ // defaultPolicy)
+ String uninitQueue = "non-initialized-queue";
+ when(applicationSubmissionContext.getQueue()).thenReturn(uninitQueue);
+ SubClusterId chosen =
+ routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue));
+
+ // empty string
+ when(applicationSubmissionContext.getQueue()).thenReturn("");
+ chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue));
+
+ // null queue also falls back to default
+ when(applicationSubmissionContext.getQueue()).thenReturn(null);
+ chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue));
+
+ }
+
+ public static SubClusterPolicyConfiguration getUniformPolicy(String queue)
+ throws FederationPolicyInitializationException {
+
+ // we go through standard lifecycle instantiating a policyManager and
+ // configuring it and serializing it to a conf.
+ UniformBroadcastPolicyManager wfp = new UniformBroadcastPolicyManager();
+ wfp.setQueue(queue);
+
+ SubClusterPolicyConfiguration fpc = wfp.serializeConf();
+
+ return fpc;
+ }
+
+ public SubClusterPolicyConfiguration getPriorityPolicy(String queue)
+ throws FederationPolicyInitializationException {
+
+ // we go through standard lifecycle instantiating a policyManager and
+ // configuring it and serializing it to a conf.
+ PriorityBroadcastPolicyManager wfp = new PriorityBroadcastPolicyManager();
+
+ // equal weight to all subcluster
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ for (SubClusterId s : subClusterIds) {
+ routerWeights.put(new SubClusterIdInfo(s), 0.9f / subClusterIds.size());
+ }
+
+ // beside the first one who gets more weight
+ SubClusterIdInfo favorite = new SubClusterIdInfo((subClusterIds.get(0)));
+ routerWeights.put(favorite, (0.1f + 0.9f / subClusterIds.size()));
+
+ WeightedPolicyInfo policyInfo = new WeightedPolicyInfo();
+ policyInfo.setRouterPolicyWeights(routerWeights);
+ wfp.setWeightedPolicyInfo(policyInfo);
+ wfp.setQueue(queue);
+
+ // serializeConf it in a context
+ SubClusterPolicyConfiguration fpc = wfp.serializeConf();
+
+ return fpc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbbeec41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
index c179521..649a61b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.federation.utils;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -29,6 +31,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHome
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -62,8 +65,8 @@ public class FederationStateStoreTestUtil {
String webAppAddress = "1.2.3.4:4";
return SubClusterInfo.newInstance(subClusterId, amRMAddress,
- clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
- CLOCK.getTime(), "capability");
+ clientRMAddress, rmAdminAddress, webAppAddress,
+ SubClusterState.SC_RUNNING, CLOCK.getTime(), "capability");
}
private void registerSubCluster(SubClusterId subClusterId)
@@ -97,6 +100,21 @@ public class FederationStateStoreTestUtil {
}
}
+ public List<SubClusterId> getAllSubClusterIds(
+ boolean filterInactiveSubclusters) throws YarnException {
+
+ List<SubClusterInfo> infos = stateStore
+ .getSubClusters(
+ GetSubClustersInfoRequest.newInstance(filterInactiveSubclusters))
+ .getSubClusters();
+ List<SubClusterId> ids = new ArrayList<>();
+ for (SubClusterInfo s : infos) {
+ ids.add(s.getSubClusterId());
+ }
+
+ return ids;
+ }
+
private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
String policyType) {
return SubClusterPolicyConfiguration.newInstance(queueName, policyType,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-3673. Create a FailoverProxy
for Federation services. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-3673. Create a FailoverProxy for Federation services. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e637a07e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e637a07e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e637a07e
Branch: refs/heads/YARN-2915
Commit: e637a07e1514ab1e6935a6a7194a8a883db54138
Parents: 6028f16
Author: Jian He <ji...@apache.org>
Authored: Mon Aug 22 14:43:07 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/conf/HAUtil.java | 30 ++-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../TestFederationRMFailoverProxyProvider.java | 154 ++++++++++++++
.../hadoop/yarn/client/ClientRMProxy.java | 4 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 23 +-
.../src/main/resources/yarn-default.xml | 7 +
.../hadoop-yarn-server-common/pom.xml | 2 -
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +-
.../failover/FederationProxyProviderUtil.java | 163 ++++++++++++++
.../FederationRMFailoverProxyProvider.java | 211 +++++++++++++++++++
.../federation/failover/package-info.java | 17 ++
12 files changed, 613 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
index e4948e7..942b08a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.conf;
-import com.google.common.annotations.VisibleForTesting;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -27,8 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.Private
public class HAUtil {
@@ -44,6 +45,29 @@ public class HAUtil {
}
/**
+ * Returns true if Federation is configured.
+ *
+ * @param conf Configuration
+ * @return true if federation is configured in the configuration; else false.
+ */
+ public static boolean isFederationEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ }
+
+ /**
+ * Returns true if RM failover is enabled in a Federation setting.
+ *
+ * @param conf Configuration
+ * @return if RM failover is enabled in conjunction with Federation in the
+ * configuration; else false.
+ */
+ public static boolean isFederationFailoverEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+ }
+
+ /**
* Returns true if Resource Manager HA is configured.
*
* @param conf Configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 72b0d26..a2c42fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2487,6 +2487,16 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+ public static final String FEDERATION_ENABLED = FEDERATION_PREFIX + "enabled";
+ public static final boolean DEFAULT_FEDERATION_ENABLED = false;
+
+ public static final String FEDERATION_FAILOVER_ENABLED =
+ FEDERATION_PREFIX + "failover.enabled";
+ public static final boolean DEFAULT_FEDERATION_FAILOVER_ENABLED = true;
+
+ public static final String FEDERATION_SUBCLUSTER_ID =
+ FEDERATION_PREFIX + "sub-cluster.id";
+
public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
FEDERATION_PREFIX + "state-store.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index bfc2534..c4d8f38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -71,6 +71,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Federation default configs to be ignored
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
new file mode 100644
index 0000000..fa3523c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for FederationRMFailoverProxyProvider.
+ */
+public class TestFederationRMFailoverProxyProvider {
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private final String dummyCapability = "cap";
+
+ @Before
+ public void setUp() throws IOException, YarnException {
+ conf = new YarnConfiguration();
+ stateStore = new MemoryFederationStateStore();
+ stateStore.init(conf);
+ FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stateStore.close();
+ stateStore = null;
+ }
+
+ @Test
+ public void testFederationRMFailoverProxyProvider() throws Exception {
+ final SubClusterId subClusterId = SubClusterId.newInstance("SC-1");
+ final MiniYARNCluster cluster = new MiniYARNCluster(
+ "testFederationRMFailoverProxyProvider", 3, 0, 1, 1);
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
+
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ 2000);
+
+ HATestUtil.setRpcAddressForRM("rm1", 10000, conf);
+ HATestUtil.setRpcAddressForRM("rm2", 20000, conf);
+ HATestUtil.setRpcAddressForRM("rm3", 30000, conf);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+
+ cluster.init(conf);
+ cluster.start();
+
+ // Transition rm3 to active;
+ makeRMActive(subClusterId, cluster, 2);
+
+ ApplicationClientProtocol client = FederationProxyProviderUtil
+ .createRMProxy(conf, ApplicationClientProtocol.class, subClusterId,
+ UserGroupInformation.getCurrentUser());
+
+ // client will retry until the rm becomes active.
+ GetClusterMetricsResponse response =
+ client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+
+ // validate response
+ checkResponse(response);
+
+ // transition rm3 to standby
+ cluster.getResourceManager(2).getRMContext().getRMAdminService()
+ .transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER));
+
+ // Transition rm2 to active;
+ makeRMActive(subClusterId, cluster, 1);
+ response = client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+
+ // validate response
+ checkResponse(response);
+
+ cluster.stop();
+ }
+
+ private void checkResponse(GetClusterMetricsResponse response) {
+ Assert.assertNotNull(response.getClusterMetrics());
+ Assert.assertEquals(0,
+ response.getClusterMetrics().getNumActiveNodeManagers());
+ }
+
+ private void makeRMActive(final SubClusterId subClusterId,
+ final MiniYARNCluster cluster, final int index) {
+ try {
+ System.out.println("Transition rm" + (index + 1) + " to active");
+ String dummyAddress = "host:" + index;
+ cluster.getResourceManager(index).getRMContext().getRMAdminService()
+ .transitionToActive(new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER));
+ ResourceManager rm = cluster.getResourceManager(index);
+ InetSocketAddress amRMAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ InetSocketAddress clientRMAddress =
+ rm.getClientRMService().getBindAddress();
+ SubClusterRegisterRequest request = SubClusterRegisterRequest
+ .newInstance(SubClusterInfo.newInstance(subClusterId,
+ amRMAddress.getAddress().getHostAddress() + ":"
+ + amRMAddress.getPort(),
+ clientRMAddress.getAddress().getHostAddress() + ":"
+ + clientRMAddress.getPort(),
+ dummyAddress, dummyAddress, SubClusterState.SC_NEW, 1,
+ dummyCapability));
+ stateStore.registerSubCluster(request);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
index b29263e..6365662 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
@@ -84,7 +84,7 @@ public class ClientRMProxy<T> extends RMProxy<T> {
@Private
@Override
- protected InetSocketAddress getRMAddress(YarnConfiguration conf,
+ public InetSocketAddress getRMAddress(YarnConfiguration conf,
Class<?> protocol) throws IOException {
if (protocol == ApplicationClientProtocol.class) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
@@ -111,7 +111,7 @@ public class ClientRMProxy<T> extends RMProxy<T> {
@Private
@Override
- protected void checkAllowedProtocols(Class<?> protocol) {
+ public void checkAllowedProtocols(Class<?> protocol) {
Preconditions.checkArgument(
protocol.isAssignableFrom(ClientRMProtocols.class),
"RM does not support this client protocol");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index 3ab06bd..f93a182 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -33,8 +33,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -64,14 +64,14 @@ public class RMProxy<T> {
* Verify the passed protocol is supported.
*/
@Private
- protected void checkAllowedProtocols(Class<?> protocol) {}
+ public void checkAllowedProtocols(Class<?> protocol) {}
/**
* Get the ResourceManager address from the provided Configuration for the
* given protocol.
*/
@Private
- protected InetSocketAddress getRMAddress(
+ public InetSocketAddress getRMAddress(
YarnConfiguration conf, Class<?> protocol) throws IOException {
throw new UnsupportedOperationException("This method should be invoked " +
"from an instance of ClientRMProxy or ServerRMProxy");
@@ -90,7 +90,8 @@ public class RMProxy<T> {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
: new YarnConfiguration(configuration);
- RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
+ RetryPolicy retryPolicy = createRetryPolicy(conf,
+ (HAUtil.isHAEnabled(conf) || HAUtil.isFederationFailoverEnabled(conf)));
return newProxyInstance(conf, protocol, instance, retryPolicy);
}
@@ -116,7 +117,7 @@ public class RMProxy<T> {
private static <T> T newProxyInstance(final YarnConfiguration conf,
final Class<T> protocol, RMProxy instance, RetryPolicy retryPolicy)
throws IOException{
- if (HAUtil.isHAEnabled(conf)) {
+ if (HAUtil.isHAEnabled(conf) || HAUtil.isFederationEnabled(conf)) {
RMFailoverProxyProvider<T> provider =
instance.createRMFailoverProxyProvider(conf, protocol);
return (T) RetryProxy.create(protocol, provider, retryPolicy);
@@ -146,7 +147,8 @@ public class RMProxy<T> {
@Deprecated
public static <T> T createRMProxy(final Configuration conf,
final Class<T> protocol, InetSocketAddress rmAddress) throws IOException {
- RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
+ RetryPolicy retryPolicy = createRetryPolicy(conf,
+ (HAUtil.isHAEnabled(conf) || HAUtil.isFederationFailoverEnabled(conf)));
T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
LOG.info("Connecting to ResourceManager at " + rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
@@ -155,9 +157,16 @@ public class RMProxy<T> {
/**
* Get a proxy to the RM at the specified address. To be used to create a
* RetryProxy.
+ *
+ * @param conf Configuration to generate retry policy
+ * @param protocol Protocol for the proxy
+ * @param rmAddress Address of the ResourceManager
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException on failure
*/
@Private
- static <T> T getProxy(final Configuration conf,
+ public static <T> T getProxy(final Configuration conf,
final Class<T> protocol, final InetSocketAddress rmAddress)
throws IOException {
return UserGroupInformation.getCurrentUser().doAs(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index bc244c9..2518dea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2612,6 +2612,13 @@
<!-- Federation Configuration -->
<property>
<description>
+ Flag to indicate whether the RM is participating in Federation or not.
+ </description>
+ <name>yarn.federation.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <description>
Machine list file to be loaded by the FederationSubCluster Resolver
</description>
<name>yarn.federation.machine-list</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index cfb2533..5ae8889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -111,12 +111,10 @@
<dependency>
<groupId>javax.cache</groupId>
<artifactId>cache-api</artifactId>
- <version>${jcache.version}</version>
</dependency>
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>ehcache</artifactId>
- <version>${ehcache.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index 8555fc3..b3038e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -71,7 +71,7 @@ public class ServerRMProxy<T> extends RMProxy<T> {
@InterfaceAudience.Private
@Override
- protected InetSocketAddress getRMAddress(YarnConfiguration conf,
+ public InetSocketAddress getRMAddress(YarnConfiguration conf,
Class<?> protocol) {
if (protocol == ResourceTracker.class) {
return conf.getSocketAddr(
@@ -93,7 +93,7 @@ public class ServerRMProxy<T> extends RMProxy<T> {
@InterfaceAudience.Private
@Override
- protected void checkAllowedProtocols(Class<?> protocol) {
+ public void checkAllowedProtocols(Class<?> protocol) {
Preconditions.checkArgument(
protocol.isAssignableFrom(ResourceTracker.class),
"ResourceManager does not support this protocol");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
new file mode 100644
index 0000000..a986008
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.failover;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class that creates proxy for specified protocols when federation is
+ * enabled. The class creates a federation aware failover provider, i.e. the
+ * failover provider uses the {@code FederationStateStore} to determine the
+ * current active ResourceManager
+ */
+@Private
+@Unstable
+public final class FederationProxyProviderUtil {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationProxyProviderUtil.class);
+
+ /**
+ * Create a proxy for the specified protocol. For non-HA, this is a direct
+ * connection to the ResourceManager address. When HA is enabled, the proxy
+ * handles the failover between the ResourceManagers as well.
+ *
+ * @param configuration Configuration to generate {@link ClientRMProxy}
+ * @param protocol Protocol for the proxy
+ * @param subClusterId the unique identifier or the sub-cluster
+ * @param user the user on whose behalf the proxy is being created
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException on failure
+ */
+ @Public
+ @Unstable
+ public static <T> T createRMProxy(Configuration configuration,
+ final Class<T> protocol, SubClusterId subClusterId,
+ UserGroupInformation user) throws IOException {
+ return createRMProxy(configuration, protocol, subClusterId, user, null);
+ }
+
+ /**
+ * Create a proxy for the specified protocol. For non-HA, this is a direct
+ * connection to the ResourceManager address. When HA is enabled, the proxy
+ * handles the failover between the ResourceManagers as well.
+ *
+ * @param configuration Configuration to generate {@link ClientRMProxy}
+ * @param protocol Protocol for the proxy
+ * @param subClusterId the unique identifier or the sub-cluster
+ * @param user the user on whose behalf the proxy is being created
+ * @param token the auth token to use for connection
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException on failure
+ */
+ @Public
+ @Unstable
+ @SuppressWarnings("unchecked")
+ public static <T> T createRMProxy(final Configuration configuration,
+ final Class<T> protocol, SubClusterId subClusterId,
+ UserGroupInformation user, final Token token) throws IOException {
+ try {
+ final YarnConfiguration conf = new YarnConfiguration(configuration);
+ updateConf(conf, subClusterId);
+ if (token != null) {
+ LOG.info(
+ "Creating RMProxy with a token: {} to subcluster: {}"
+ + " for protocol: {}",
+ token, subClusterId, protocol.getSimpleName());
+ user.addToken(token);
+ setAuthModeInConf(conf);
+ } else {
+ LOG.info("Creating RMProxy without a token to subcluster: {}"
+ + " for protocol: {}", subClusterId, protocol.getSimpleName());
+ }
+ final T proxyConnection = user.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws Exception {
+ return ClientRMProxy.createRMProxy(conf, protocol);
+ }
+ });
+
+ return proxyConnection;
+ } catch (IOException e) {
+ String message =
+ "Error while creating of RM application master service proxy for"
+ + " appAttemptId: " + user;
+ LOG.info(message);
+ throw new YarnRuntimeException(message, e);
+ } catch (InterruptedException e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+
+ private static void setAuthModeInConf(Configuration conf) {
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ SaslRpcServer.AuthMethod.TOKEN.toString());
+ }
+
+ // updating the conf with the refreshed RM addresses as proxy creations
+ // are based out of conf
+ private static void updateConf(Configuration conf,
+ SubClusterId subClusterId) {
+ conf.set(YarnConfiguration.FEDERATION_SUBCLUSTER_ID, subClusterId.getId());
+ // In a Federation setting, we will connect to not just the local cluster RM
+ // but also multiple external RMs. The membership information of all the RMs
+ // that are currently
+ // participating in Federation is available in the central
+ // FederationStateStore.
+ // So we will:
+ // 1. obtain the RM service addresses from FederationStateStore using the
+ // FederationRMFailoverProxyProvider.
+ // 2. disable traditional HA as that depends on local configuration lookup
+ // for RMs using indexes.
+ // 3. we will enable federation failover IF traditional HA is enabled so
+ // that the appropriate failover RetryPolicy is initialized.
+ conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+ conf.setClass(YarnConfiguration.CLIENT_FAILOVER_PROXY_PROVIDER,
+ FederationRMFailoverProxyProvider.class, RMFailoverProxyProvider.class);
+ if (HAUtil.isHAEnabled(conf)) {
+ conf.setBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+ }
+ }
+
+ // disable instantiation
+ private FederationProxyProviderUtil() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
new file mode 100644
index 0000000..90a9239
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.failover;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
+import org.apache.hadoop.yarn.client.RMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A FailoverProxyProvider implementation that uses the
+ * {@code FederationStateStore} to determine the ResourceManager to connect to.
+ * This supports both HA and regular mode which is controlled by configuration.
+ */
+@Private
+@Unstable
+public class FederationRMFailoverProxyProvider<T>
+ implements RMFailoverProxyProvider<T> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationRMFailoverProxyProvider.class);
+
+ private RMProxy<T> rmProxy;
+ private Class<T> protocol;
+ private T current;
+ private YarnConfiguration conf;
+ private FederationStateStoreFacade facade;
+ private SubClusterId subClusterId;
+ private Collection<Token<? extends TokenIdentifier>> originalTokens;
+ private boolean federationFailoverEnabled = false;
+
+ @Override
+ public void init(Configuration configuration, RMProxy<T> proxy,
+ Class<T> proto) {
+ this.rmProxy = proxy;
+ this.protocol = proto;
+ this.rmProxy.checkAllowedProtocols(this.protocol);
+ String clusterId =
+ configuration.get(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
+ Preconditions.checkNotNull(clusterId, "Missing Federation SubClusterId");
+ this.subClusterId = SubClusterId.newInstance(clusterId);
+ this.facade = facade.getInstance();
+ if (configuration instanceof YarnConfiguration) {
+ this.conf = (YarnConfiguration) configuration;
+ }
+ federationFailoverEnabled =
+ conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
+ conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES,
+ YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES));
+
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ conf.getInt(
+ YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS,
+ YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
+
+ try {
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ originalTokens = currentUser.getTokens();
+ LOG.info("Initialized Federation proxy for user: {}",
+ currentUser.getUserName());
+ } catch (IOException e) {
+ LOG.warn("Could not get information of requester, ignoring for now.");
+ }
+
+ }
+
+ private void addOriginalTokens(UserGroupInformation currentUser) {
+ if (originalTokens == null || originalTokens.isEmpty()) {
+ return;
+ }
+ for (Token<? extends TokenIdentifier> token : originalTokens) {
+ currentUser.addToken(token);
+ }
+ }
+
+ private T getProxyInternal(boolean isFailover) {
+ SubClusterInfo subClusterInfo;
+ UserGroupInformation currentUser = null;
+ try {
+ LOG.info("Failing over to the ResourceManager for SubClusterId: {}",
+ subClusterId);
+ subClusterInfo = facade.getSubCluster(subClusterId, isFailover);
+ // updating the conf with the refreshed RM addresses as proxy
+ // creations
+ // are based out of conf
+ updateRMAddress(subClusterInfo);
+ currentUser = UserGroupInformation.getCurrentUser();
+ addOriginalTokens(currentUser);
+ } catch (YarnException e) {
+ LOG.error("Exception while trying to create proxy to the ResourceManager"
+ + " for SubClusterId: {}", subClusterId, e);
+ return null;
+ } catch (IOException e) {
+ LOG.warn("Could not get information of requester, ignoring for now.");
+ }
+ try {
+ final InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
+ LOG.info("Connecting to {} with protocol {} as user: {}", rmAddress,
+ protocol.getSimpleName(), currentUser);
+ LOG.info("Failed over to the RM at {} for SubClusterId: {}", rmAddress,
+ subClusterId);
+ return RMProxy.getProxy(conf, protocol, rmAddress);
+ } catch (IOException ioe) {
+ LOG.error(
+ "IOException while trying to create proxy to the ResourceManager"
+ + " for SubClusterId: {}",
+ subClusterId, ioe);
+ return null;
+ }
+ }
+
+ private void updateRMAddress(SubClusterInfo subClusterInfo) {
+ if (subClusterInfo != null) {
+ if (protocol == ApplicationClientProtocol.class) {
+ conf.set(YarnConfiguration.RM_ADDRESS,
+ subClusterInfo.getClientRMServiceAddress());
+ } else if (protocol == ApplicationMasterProtocol.class) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ subClusterInfo.getAMRMServiceAddress());
+ } else if (protocol == ResourceManagerAdministrationProtocol.class) {
+ conf.set(YarnConfiguration.RM_ADMIN_ADDRESS,
+ subClusterInfo.getRMAdminServiceAddress());
+ }
+ }
+ }
+
+ @Override
+ public synchronized ProxyInfo<T> getProxy() {
+ if (current == null) {
+ current = getProxyInternal(false);
+ }
+ return new ProxyInfo<T>(current, subClusterId.getId());
+ }
+
+ @Override
+ public synchronized void performFailover(T currentProxy) {
+ closeInternal(currentProxy);
+ current = getProxyInternal(federationFailoverEnabled);
+ }
+
+ @Override
+ public Class<T> getInterface() {
+ return protocol;
+ }
+
+ private void closeInternal(T currentProxy) {
+ if ((currentProxy != null) && (currentProxy instanceof Closeable)) {
+ try {
+ ((Closeable) currentProxy).close();
+ } catch (IOException e) {
+ LOG.warn("Exception while trying to close proxy", e);
+ }
+ } else {
+ RPC.stopProxy(currentProxy);
+ }
+
+ }
+
+ /**
+ * Close all the proxy objects which have been opened over the lifetime of
+ * this proxy provider.
+ */
+ @Override
+ public synchronized void close() throws IOException {
+ closeInternal(current);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e637a07e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java
new file mode 100644
index 0000000..b1baa0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.failover;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-5467. InputValidator for the
FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6028f167/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
new file mode 100644
index 0000000..13175ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
@@ -0,0 +1,1265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for FederationApplicationInputValidator,
+ * FederationMembershipInputValidator, and FederationPolicyInputValidator.
+ */
+public class TestFederationStateStoreInputValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationStateStoreInputValidator.class);
+
+ private static SubClusterId subClusterId;
+ private static String amRMServiceAddress;
+ private static String clientRMServiceAddress;
+ private static String rmAdminServiceAddress;
+ private static String rmWebServiceAddress;
+ private static int lastHeartBeat;
+ private static SubClusterState stateNew;
+ private static SubClusterState stateLost;
+ private static ApplicationId appId;
+ private static int lastStartTime;
+ private static String capability;
+ private static String queue;
+ private static String type;
+ private static ByteBuffer params;
+
+ private static SubClusterId subClusterIdInvalid;
+ private static SubClusterId subClusterIdNull;
+
+ private static int lastHeartBeatNegative;
+ private static int lastStartTimeNegative;
+
+ private static SubClusterState stateNull;
+ private static ApplicationId appIdNull;
+
+ private static String capabilityNull;
+ private static String capabilityEmpty;
+
+ private static String addressNull;
+ private static String addressEmpty;
+ private static String addressWrong;
+ private static String addressWrongPort;
+
+ private static String queueEmpty;
+ private static String queueNull;
+
+ private static String typeEmpty;
+ private static String typeNull;
+
+ @BeforeClass
+ public static void setUp() {
+ subClusterId = SubClusterId.newInstance("abc");
+ amRMServiceAddress = "localhost:8032";
+ clientRMServiceAddress = "localhost:8034";
+ rmAdminServiceAddress = "localhost:8031";
+ rmWebServiceAddress = "localhost:8088";
+ lastHeartBeat = 1000;
+ stateNew = SubClusterState.SC_NEW;
+ stateLost = SubClusterState.SC_LOST;
+ lastStartTime = 1000;
+ capability = "Memory VCores";
+ appId = ApplicationId.newInstance(lastStartTime, 1);
+ queue = "default";
+ type = "random";
+ params = ByteBuffer.allocate(10);
+ params.put((byte) 0xFF);
+
+ subClusterIdInvalid = SubClusterId.newInstance("");
+ subClusterIdNull = null;
+
+ lastHeartBeatNegative = -10;
+ lastStartTimeNegative = -10;
+
+ stateNull = null;
+ appIdNull = null;
+
+ capabilityNull = null;
+ capabilityEmpty = "";
+
+ addressNull = null;
+ addressEmpty = "";
+ addressWrong = "AddressWrong";
+ addressWrongPort = "Address:WrongPort";
+
+ queueEmpty = "";
+ queueNull = null;
+
+ typeEmpty = "";
+ typeNull = null;
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequest() {
+
+ // Execution with valid inputs
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SubClusterRegisterRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterRegister Request."));
+ }
+
+ // Execution with null SubClusterInfo
+
+ subClusterInfo = null;
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Information."));
+ }
+
+ // Execution with Null SubClusterId
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterIdNull, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with Invalid SubClusterId
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterIdInvalid, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with Null State
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster State information."));
+ }
+
+ // Execution with Null Capability
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capabilityNull);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+
+ // Execution with Empty Capability
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capabilityEmpty);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequestTimestamp() {
+
+ // Execution with Negative Last Heartbeat
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeatNegative, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid timestamp information."));
+ }
+
+ // Execution with Negative Last StartTime
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTimeNegative, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid timestamp information."));
+ }
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequestAddress() {
+ // Execution with Null Address for amRMServiceAddress
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, addressNull,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for amRMServiceAddress
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId, addressEmpty,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Null Address for clientRMServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressNull, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for clientRMServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressEmpty, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Null Address for rmAdminServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressNull, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for rmAdminServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressEmpty, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Null Address for rmWebServiceAddress
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressNull, lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for rmWebServiceAddress
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressEmpty, lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequestAddressInvalid() {
+
+ // Address is not in host:port format for amRMService
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, addressWrong,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Address is not in host:port format for clientRMService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressWrong, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Address is not in host:port format for rmAdminService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressWrong, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Address is not in host:port format for rmWebService
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressWrong, lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for amRMService
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId, addressWrongPort,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for clientRMService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressWrongPort, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for rmAdminService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressWrongPort, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for rmWebService
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressWrongPort, lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ }
+
+ @Test
+ public void testValidateSubClusterDeregisterRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterId, stateLost);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SubClusterDeregisterRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterDeregister Request."));
+ }
+
+ // Execution with null SubClusterId
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterIdNull, stateLost);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ try {
+ SubClusterDeregisterRequest request = SubClusterDeregisterRequest
+ .newInstance(subClusterIdInvalid, stateLost);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with null SubClusterState
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterId, stateNull);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster State information."));
+ }
+
+ // Execution with invalid SubClusterState
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterId, stateNew);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().startsWith("Invalid non-final state: "));
+ }
+ }
+
+ @Test
+ public void testSubClusterHeartbeatRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SubClusterHeartbeatRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterHeartbeat Request."));
+ }
+
+ // Execution with null SubClusterId
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterIdNull, lastHeartBeat, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ try {
+ SubClusterHeartbeatRequest request =
+ SubClusterHeartbeatRequest.newInstance(subClusterIdInvalid,
+ lastHeartBeat, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with null SubClusterState
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateNull, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster State information."));
+ }
+
+ // Execution with negative Last Heartbeat
+
+ try {
+ SubClusterHeartbeatRequest request =
+ SubClusterHeartbeatRequest.newInstance(subClusterId,
+ lastHeartBeatNegative, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid timestamp information."));
+ }
+
+ // Execution with null Capability
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateLost, capabilityNull);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+
+ // Execution with empty Capability
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateLost, capabilityEmpty);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterInfoRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ GetSubClusterInfoRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing GetSubClusterInfo Request."));
+ }
+
+ // Execution with null SubClusterId
+
+ try {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterIdNull);
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ try {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterIdInvalid);
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+ }
+
+ @Test
+ public void testAddApplicationHomeSubClusterRequest() {
+
+ // Execution with valid inputs
+
+ ApplicationHomeSubCluster applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ AddApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing AddApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationHomeSubCluster
+
+ applicationHomeSubCluster = null;
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing ApplicationHomeSubCluster Info."));
+ }
+
+ // Execution with null SubClusterId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdNull);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdInvalid);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with Null ApplicationId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appIdNull, subClusterId);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterRequest() {
+
+ // Execution with valid inputs
+
+ ApplicationHomeSubCluster applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ UpdateApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing UpdateApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationHomeSubCluster
+
+ applicationHomeSubCluster = null;
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing ApplicationHomeSubCluster Info."));
+ }
+
+ // Execution with null SubClusteId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdNull);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdInvalid);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with null ApplicationId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appIdNull, subClusterId);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ GetApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing GetApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationId
+
+ try {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appIdNull);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterRequestNull() {
+
+ // Execution with valid inputs
+
+ try {
+ DeleteApplicationHomeSubClusterRequest request =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ DeleteApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing DeleteApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationId
+
+ try {
+ DeleteApplicationHomeSubClusterRequest request =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appIdNull);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+
+ }
+
+ @Test
+ public void testGetSubClusterPolicyConfigurationRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue);
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request = null;
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing GetSubClusterPolicyConfiguration Request."));
+ }
+
+ // Execution with null queue id
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queueNull);
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ // Execution with empty queue id
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queueEmpty);
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ }
+
+ @Test
+ public void testSetSubClusterPolicyConfigurationRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queue, type, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SetSubClusterPolicyConfigurationRequest request = null;
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SetSubClusterPolicyConfiguration Request."));
+ }
+
+ // Execution with null SubClusterPolicyConfiguration
+
+ try {
+ SubClusterPolicyConfiguration policy = null;
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterPolicyConfiguration."));
+ }
+
+ // Execution with null queue id
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queueNull, type, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ // Execution with empty queue id
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queueEmpty, type, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ // Execution with null policy type
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queue, typeNull, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Policy Type."));
+ }
+
+ // Execution with empty policy type
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queue, typeEmpty, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Policy Type."));
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-5408. Compose Federation
membership/application/policy APIs into an uber FederationStateStore API.
(Ellen Hui via Subru).
Posted by su...@apache.org.
YARN-5408. Compose Federation membership/application/policy APIs into an uber FederationStateStore API. (Ellen Hui via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/630cb6c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/630cb6c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/630cb6c2
Branch: refs/heads/YARN-2915
Commit: 630cb6c272b893e98fa6595a1d1fd50a0c842f7b
Parents: edbcd9b
Author: Subru Krishnan <su...@apache.org>
Authored: Mon Aug 8 14:53:38 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
...ederationApplicationHomeSubClusterStore.java | 18 ++----
.../store/FederationMembershipStateStore.java | 14 +----
.../federation/store/FederationStateStore.java | 64 ++++++++++++++++++++
.../store/impl/MemoryFederationStateStore.java | 19 ------
.../impl/FederationStateStoreBaseTest.java | 57 +++++++++--------
.../impl/TestMemoryFederationStateStore.java | 21 +------
6 files changed, 99 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/630cb6c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
index 217ee2e..22bb88a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHom
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
/**
* FederationApplicationHomeSubClusterStore maintains the state of all
@@ -50,15 +49,6 @@ import org.apache.hadoop.yarn.server.records.Version;
public interface FederationApplicationHomeSubClusterStore {
/**
- * Get the {@link Version} of the underlying federation application state
- * store.
- *
- * @return the {@link Version} of the underlying federation application state
- * store
- */
- Version getApplicationStateStoreVersion();
-
- /**
* Register the home {@code SubClusterId} of the newly submitted
* {@code ApplicationId}. Currently response is empty if the operation was
* successful, if not an exception reporting reason for a failure.
@@ -91,16 +81,16 @@ public interface FederationApplicationHomeSubClusterStore {
* {@code ApplicationId}.
*
* @param request contains the application queried
- * @return {@code ApplicationHomeSubCluster} containing the application's
- * home subcluster
+ * @return {@code ApplicationHomeSubCluster} containing the application's home
+ * subcluster
* @throws YarnException if the request is invalid/fails
*/
GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
GetApplicationHomeSubClusterRequest request) throws YarnException;
/**
- * Get the {@code ApplicationHomeSubCluster} list representing the mapping
- * of all submitted applications to it's home sub-cluster.
+ * Get the {@code ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
*
* @param request empty representing all applications
* @return the mapping of all submitted application to it's home sub-cluster
http://git-wip-us.apache.org/repos/asf/hadoop/blob/630cb6c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
index 378eadc..7778d5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbea
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
/**
* FederationMembershipStateStore maintains the state of all
@@ -42,15 +41,6 @@ import org.apache.hadoop.yarn.server.records.Version;
public interface FederationMembershipStateStore {
/**
- * Get the {@link Version} of the underlying federation membership state
- * store.
- *
- * @return the {@link Version} of the underlying federation membership state
- * store
- */
- Version getMembershipStateStoreVersion();
-
- /**
* Register a <em>subcluster</em> by publishing capabilities as represented by
* {@code SubClusterInfo} to indicate participation in federation. This is
* typically done during initialization or restart/failover of the
@@ -80,7 +70,7 @@ public interface FederationMembershipStateStore {
*/
SubClusterDeregisterResponse deregisterSubCluster(
SubClusterDeregisterRequest subClusterDeregisterRequest)
- throws YarnException;
+ throws YarnException;
/**
* Periodic heartbeat from a <code>ResourceManager</code> participating in
@@ -96,7 +86,7 @@ public interface FederationMembershipStateStore {
*/
SubClusterHeartbeatResponse subClusterHeartbeat(
SubClusterHeartbeatRequest subClusterHeartbeatRequest)
- throws YarnException;
+ throws YarnException;
/**
* Get the membership information of <em>subcluster</em> as identified by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/630cb6c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
new file mode 100644
index 0000000..9397e9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.records.Version;
+
+/**
+ * FederationStore extends the three interfaces used to coordinate the state of
+ * a federated cluster: {@link FederationApplicationHomeSubClusterStore},
+ * {@link FederationMembershipStateStore}, and {@link FederationPolicyStore}.
+ *
+ */
+public interface FederationStateStore
+ extends FederationApplicationHomeSubClusterStore,
+ FederationMembershipStateStore, FederationPolicyStore {
+
+ /**
+ * Initialize the FederationStore.
+ *
+ * @param conf the cluster configuration
+ * @throws YarnException if initialization fails
+ */
+ void init(Configuration conf) throws YarnException;
+
+ /**
+ * Perform any cleanup operations of the StateStore.
+ *
+ * @throws Exception if cleanup fails
+ */
+ void close() throws Exception;
+
+ /**
+ * Get the {@link Version} of the underlying federation state store client.
+ *
+ * @return the {@link Version} of the underlying federation store client
+ */
+ Version getCurrentVersion();
+
+ /**
+ * Load the version information from the federation state store.
+ *
+ * @return the {@link Version} of the federation state store
+ */
+ Version loadVersion();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/630cb6c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 7fdc4a9..cea4ac2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -36,11 +36,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbea
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* In-memory implementation of FederationMembershipStateStore.
*/
@@ -52,11 +49,6 @@ public class MemoryFederationStateStore
private final MonotonicClock clock = new MonotonicClock();
@Override
- public Version getMembershipStateStoreVersion() {
- return null;
- }
-
- @Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
@@ -122,17 +114,6 @@ public class MemoryFederationStateStore
}
}
return GetSubClustersInfoResponse.newInstance(result);
-
- }
-
- @VisibleForTesting
- public Map<SubClusterId, SubClusterInfo> getMembershipTable() {
- return membership;
- }
-
- @VisibleForTesting
- public void clearMembershipTable() {
- membership.clear();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/630cb6c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 7eb1c86..c76a485 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -19,25 +19,21 @@ package org.apache.hadoop.yarn.server.federation.store.impl;
import java.io.IOException;
-import org.junit.Before;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
/**
@@ -45,17 +41,22 @@ import org.junit.Test;
*/
public abstract class FederationStateStoreBaseTest {
- static final Logger LOG =
- LoggerFactory.getLogger(FederationStateStoreBaseTest.class);
private static final MonotonicClock CLOCK = new MonotonicClock();
- private FederationMembershipStateStore stateStore = getStateStore();
+ private FederationMembershipStateStore stateStore;
@Before
public void before() throws IOException {
- clearMembership();
+ stateStore = getCleanStateStore();
}
+ @After
+ public void after() {
+ stateStore = null;
+ }
+
+ protected abstract FederationMembershipStateStore getCleanStateStore();
+
@Test
public void testRegisterSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
@@ -63,11 +64,9 @@ public abstract class FederationStateStoreBaseTest {
SubClusterRegisterResponse result = stateStore.registerSubCluster(
SubClusterRegisterRequest.newInstance(subClusterInfo));
- Map<SubClusterId, SubClusterInfo> membership = getMembership();
- Assert.assertNotNull(membership.get(subClusterId));
Assert.assertNotNull(result);
- Assert.assertEquals(subClusterInfo, membership.get(subClusterId));
+ Assert.assertEquals(subClusterInfo, querySubClusterInfo(subClusterId));
}
@Test
@@ -83,10 +82,8 @@ public abstract class FederationStateStoreBaseTest {
stateStore.deregisterSubCluster(deregisterRequest);
- Map<SubClusterId, SubClusterInfo> membership = getMembership();
- Assert.assertNotNull(membership.get(subClusterId));
- Assert.assertEquals(membership.get(subClusterId).getState(),
- SubClusterState.SC_UNREGISTERED);
+ Assert.assertEquals(SubClusterState.SC_UNREGISTERED,
+ querySubClusterInfo(subClusterId).getState());
}
@Test
@@ -179,10 +176,9 @@ public abstract class FederationStateStoreBaseTest {
.newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
stateStore.subClusterHeartbeat(heartbeatRequest);
- Map<SubClusterId, SubClusterInfo> membership = getMembership();
- Assert.assertEquals(membership.get(subClusterId).getState(),
- SubClusterState.SC_RUNNING);
- Assert.assertNotNull(membership.get(subClusterId).getLastHeartBeat());
+ Assert.assertEquals(SubClusterState.SC_RUNNING,
+ querySubClusterInfo(subClusterId).getState());
+ Assert.assertNotNull(querySubClusterInfo(subClusterId).getLastHeartBeat());
}
@Test
@@ -212,10 +208,11 @@ public abstract class FederationStateStoreBaseTest {
CLOCK.getTime(), "cabability");
}
- protected abstract Map<SubClusterId, SubClusterInfo> getMembership();
-
- protected abstract void clearMembership();
-
- protected abstract FederationMembershipStateStore getStateStore();
+ private SubClusterInfo querySubClusterInfo(SubClusterId subClusterId)
+ throws YarnException {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ return stateStore.getSubCluster(request).getSubClusterInfo();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/630cb6c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
index b74ffbd..9396eda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -17,11 +17,7 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
-import java.util.Map;
-
import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
/**
* Unit tests for MemoryFederationStateStore.
@@ -29,21 +25,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
public class TestMemoryFederationStateStore
extends FederationStateStoreBaseTest {
- private static final MemoryFederationStateStore STATESTORE =
- new MemoryFederationStateStore();
-
- @Override
- protected Map<SubClusterId, SubClusterInfo> getMembership() {
- return STATESTORE.getMembershipTable();
- }
-
- @Override
- protected void clearMembership() {
- STATESTORE.clearMembershipTable();
- }
-
@Override
- protected FederationMembershipStateStore getStateStore() {
- return STATESTORE;
+ protected FederationMembershipStateStore getCleanStateStore() {
+ return new MemoryFederationStateStore();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDFS-11409. DatanodeInfo
getNetworkLocation and setNetworkLocation shoud use volatile instead of
synchronized. Contributed by Chen Liang.
Posted by su...@apache.org.
HDFS-11409. DatanodeInfo getNetworkLocation and setNetworkLocation shoud use volatile instead of synchronized. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaf27132
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaf27132
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaf27132
Branch: refs/heads/YARN-2915
Commit: aaf27132350547fcde1fdb372f19626838f44bc4
Parents: 0cf5993
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Feb 14 12:52:34 2017 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Feb 14 12:52:34 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf27132/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index 41735b1..acbcffa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -51,7 +51,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
private long lastUpdate;
private long lastUpdateMonotonic;
private int xceiverCount;
- private String location = NetworkTopology.DEFAULT_RACK;
+ private volatile String location = NetworkTopology.DEFAULT_RACK;
private String softwareVersion;
private List<String> dependentHostNames = new LinkedList<>();
private String upgradeDomain;
@@ -293,11 +293,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
/** network location */
@Override
- public synchronized String getNetworkLocation() {return location;}
+ public String getNetworkLocation() {return location;}
/** Sets the network location */
@Override
- public synchronized void setNetworkLocation(String location) {
+ public void setNetworkLocation(String location) {
this.location = NodeBase.normalize(location);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-5324. Stateless Federation
router policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5324. Stateless Federation router policies implementation. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cf5730c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cf5730c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cf5730c
Branch: refs/heads/YARN-2915
Commit: 6cf5730c1b40f87a78050ef71b4b9129e6b2963b
Parents: 288c55e
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Sep 22 17:06:57 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../policies/FederationPolicyConfigurator.java | 91 -------
.../FederationPolicyInitializationContext.java | 11 +-
.../policies/FederationPolicyManager.java | 126 +++++++++
.../policies/FederationPolicyWriter.java | 45 ----
.../policies/dao/WeightedPolicyInfo.java | 253 +++++++++++++++++++
.../federation/policies/dao/package-info.java | 20 ++
.../router/BaseWeightedRouterPolicy.java | 150 +++++++++++
.../policies/router/LoadBasedRouterPolicy.java | 109 ++++++++
.../policies/router/PriorityRouterPolicy.java | 66 +++++
.../router/UniformRandomRouterPolicy.java | 85 +++++++
.../router/WeightedRandomRouterPolicy.java | 79 ++++++
.../store/records/SubClusterIdInfo.java | 75 ++++++
.../policies/BaseFederationPoliciesTest.java | 155 ++++++++++++
...ionPolicyInitializationContextValidator.java | 17 +-
.../router/TestLoadBasedRouterPolicy.java | 109 ++++++++
.../router/TestPriorityRouterPolicy.java | 87 +++++++
.../router/TestUniformRandomRouterPolicy.java | 65 +++++
.../router/TestWeightedRandomRouterPolicy.java | 127 ++++++++++
.../utils/FederationPoliciesTestUtil.java | 82 +++++-
19 files changed, 1604 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
deleted file mode 100644
index fdc3857..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-
-
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-
-import org.apache.hadoop.yarn.server.federation.policies.router
- .FederationRouterPolicy;
-
-/**
- * Implementors of this interface are capable to instantiate and (re)initalize
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} based on
- * a {@link FederationPolicyInitializationContext}. The reason to bind these two
- * policies together is to make sure we remain consistent across the router and
- * amrmproxy policy decisions.
- */
-public interface FederationPolicyConfigurator {
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
- * current context, otherwise a new instance initialized with the current
- * context is provided. If the instance is compatible with the current class
- * the implementors should attempt to reinitalize (retaining state). To affect
- * a complete policy reset oldInstance should be null.
- *
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
- *
- * @return an updated {@link FederationAMRMProxyPolicy
- }.
- *
- * @throws FederationPolicyInitializationException if the initialization
- * cannot be completed
- * properly. The oldInstance
- * should be still valid in
- * case of failed
- * initialization.
- */
- FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
- FederationAMRMProxyPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationRouterPolicy} reinitialized with the current
- * context, otherwise a new instance initialized with the current context is
- * provided. If the instance is compatible with the current class the
- * implementors should attempt to reinitalize (retaining state). To affect a
- * complete policy reset oldInstance shoulb be set to null.
- *
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
- *
- * @return an updated {@link FederationRouterPolicy}.
- *
- * @throws FederationPolicyInitializationException if the initalization cannot
- * be completed properly. The
- * oldInstance should be still
- * valid in case of failed
- * initialization.
- */
- FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
- FederationRouterPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
index 879ccee..9347fd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -59,13 +59,12 @@ public class FederationPolicyInitializationContext {
/**
* Setter for the {@link SubClusterPolicyConfiguration}.
*
- * @param federationPolicyConfiguration the
- * {@link SubClusterPolicyConfiguration}
- * to be used for initialization.
+ * @param fedPolicyConfiguration the {@link SubClusterPolicyConfiguration}
+ * to be used for initialization.
*/
- public void setFederationPolicyConfiguration(
- SubClusterPolicyConfiguration federationPolicyConfiguration) {
- this.federationPolicyConfiguration = federationPolicyConfiguration;
+ public void setSubClusterPolicyConfiguration(
+ SubClusterPolicyConfiguration fedPolicyConfiguration) {
+ this.federationPolicyConfiguration = fedPolicyConfiguration;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
new file mode 100644
index 0000000..e5dba63
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+/**
+ *
+ * Implementors need to provide the ability to serliaze a policy and its
+ * configuration as a {@link SubClusterPolicyConfiguration}, as well as
+ * provide (re)initialization mechanics for the underlying
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
+ *
+ * The serialization aspects are used by admin APIs or a policy engine to
+ * store a serialized configuration in the {@code FederationStateStore},
+ * while the getters methods are used to obtain a propertly inizialized
+ * policy in the {@code Router} and {@code AMRMProxy} respectively.
+ *
+ * This interface by design binds together
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} and
+ * provide lifecycle support for serialization and deserialization, to reduce
+ * configuration mistakes (combining incompatible policies).
+ *
+ */
+public interface FederationPolicyManager {
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
+ * current context, otherwise a new instance initialized with the current
+ * context is provided. If the instance is compatible with the current class
+ * the implementors should attempt to reinitalize (retaining state). To affect
+ * a complete policy reset oldInstance should be null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationAMRMProxyPolicy
+ }.
+ *
+ * @throws FederationPolicyInitializationException if the initialization
+ * cannot be completed
+ * properly. The oldInstance
+ * should be still valid in
+ * case of failed
+ * initialization.
+ */
+ FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationRouterPolicy} reinitialized with the current
+ * context, otherwise a new instance initialized with the current context is
+ * provided. If the instance is compatible with the current class the
+ * implementors should attempt to reinitalize (retaining state). To affect a
+ * complete policy reset oldInstance shoulb be set to null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationRouterPolicy}.
+ *
+ * @throws FederationPolicyInitializationException if the initalization cannot
+ * be completed properly. The
+ * oldInstance should be still
+ * valid in case of failed
+ * initialization.
+ */
+ FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
+ * This is to be used when writing a policy object in the federation policy
+ * store.
+ *
+ * @return a valid policy configuration representing this object
+ * parametrization.
+ *
+ * @throws FederationPolicyInitializationException if the current state cannot
+ * be serialized properly
+ */
+ SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException;
+
+
+ /**
+ * This method returns the queue this policy is configured for.
+ * @return the name of the queue.
+ */
+ String getQueue();
+
+ /**
+ * This methods provides a setter for the queue this policy is specified for.
+ * @param queue the name of the queue.
+ */
+ void setQueue(String queue);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
deleted file mode 100644
index 5034b7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-/**
- * Implementors of this class are able to serializeConf the configuraiton of a
- * policy as a {@link SubClusterPolicyConfiguration}. This is used during the
- * lifetime of a policy from the admin APIs or policy engine to serializeConf
- * the policy into the policy store.
- */
-public interface FederationPolicyWriter {
-
- /**
- /**
- * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
- * This is to be used when writing a policy object in the federation policy
- * store.
- *
- * @return a valid policy configuration representing this object
- * parametrization.
- *
- * @throws FederationPolicyInitializationException if the current state cannot
- * be serialized properly
- */
- SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
new file mode 100644
index 0000000..a0fa37f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.dao;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This is a DAO class for the configuration of parameteres for federation
+ * policies. This generalizes several possible configurations as two lists of
+ * {@link SubClusterIdInfo} and corresponding weights as a
+ * {@link Float}. The interpretation of the weight is left to the logic in
+ * the policy.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@XmlRootElement(name = "federation-policy")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class WeightedPolicyInfo {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WeightedPolicyInfo.class);
+
+ private Map<SubClusterIdInfo, Float> routerPolicyWeights = new HashMap<>();
+ private Map<SubClusterIdInfo, Float> amrmPolicyWeights = new HashMap<>();
+ private float headroomAlpha;
+
+ private static JSONJAXBContext jsonjaxbContext = initContext();
+
+ private static JSONJAXBContext initContext() {
+ try {
+ return new JSONJAXBContext(JSONConfiguration.DEFAULT,
+ WeightedPolicyInfo.class);
+ } catch (JAXBException e) {
+ LOG.error("Error parsing the policy.", e);
+ }
+ return null;
+ }
+
+ public WeightedPolicyInfo() {
+ //JAXB needs this
+ }
+
+ /**
+ * Setter method for Router weights.
+ *
+ * @param policyWeights the router weights.
+ */
+ public void setRouterPolicyWeights(
+ Map<SubClusterIdInfo, Float> policyWeights) {
+ this.routerPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Setter method for ARMRMProxy weights.
+ *
+ * @param policyWeights the amrmproxy weights.
+ */
+ public void setAMRMPolicyWeights(
+ Map<SubClusterIdInfo, Float> policyWeights) {
+ this.amrmPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Getter of the router weights.
+ * @return the router weights.
+ */
+ public Map<SubClusterIdInfo, Float> getRouterPolicyWeights() {
+ return routerPolicyWeights;
+ }
+
+ /**
+ * Getter for AMRMProxy weights.
+ * @return the AMRMProxy weights.
+ */
+ public Map<SubClusterIdInfo, Float> getAMRMPolicyWeights() {
+ return amrmPolicyWeights;
+ }
+
+ /**
+ * Deserializes a {@link WeightedPolicyInfo} from a byte UTF-8 JSON
+ * representation.
+ *
+ * @param bb the input byte representation.
+ *
+ * @return the {@link WeightedPolicyInfo} represented.
+ *
+ * @throws FederationPolicyInitializationException if a deserializaiton error
+ * occurs.
+ */
+ public static WeightedPolicyInfo fromByteBuffer(ByteBuffer bb)
+ throws FederationPolicyInitializationException {
+
+ if (jsonjaxbContext == null) {
+ throw new FederationPolicyInitializationException("JSONJAXBContext should"
+ + " not be null.");
+ }
+
+ try {
+ JSONUnmarshaller unmarshaller = jsonjaxbContext.createJSONUnmarshaller();
+ final byte[] bytes = new byte[bb.remaining()];
+ bb.get(bytes);
+ String params = new String(bytes, Charset.forName("UTF-8"));
+
+ WeightedPolicyInfo weightedPolicyInfo = unmarshaller
+ .unmarshalFromJSON(new StringReader(params),
+ WeightedPolicyInfo.class);
+ return weightedPolicyInfo;
+ } catch (JAXBException j) {
+ throw new FederationPolicyInitializationException(j);
+ }
+ }
+
+ /**
+ * Converts the policy into a byte array representation in the input {@link
+ * ByteBuffer}.
+ *
+ * @return byte array representation of this policy configuration.
+ *
+ * @throws FederationPolicyInitializationException if a serialization error
+ * occurs.
+ */
+ public ByteBuffer toByteBuffer()
+ throws FederationPolicyInitializationException {
+ if (jsonjaxbContext == null) {
+ throw new FederationPolicyInitializationException("JSONJAXBContext should"
+ + " not be null.");
+ }
+ try {
+ String s = toJSONString();
+ return ByteBuffer.wrap(s.getBytes(Charset.forName("UTF-8")));
+ } catch (JAXBException j) {
+ throw new FederationPolicyInitializationException(j);
+ }
+ }
+
+ private String toJSONString() throws JAXBException {
+ JSONMarshaller marshaller = jsonjaxbContext.createJSONMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ StringWriter sw = new StringWriter(256);
+ marshaller.marshallToJSON(this, sw);
+ return sw.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+
+ if (other == null || !other.getClass().equals(this.getClass())) {
+ return false;
+ }
+
+ WeightedPolicyInfo otherPolicy =
+ (WeightedPolicyInfo) other;
+ Map<SubClusterIdInfo, Float> otherAMRMWeights =
+ otherPolicy.getAMRMPolicyWeights();
+ Map<SubClusterIdInfo, Float> otherRouterWeights =
+ otherPolicy.getRouterPolicyWeights();
+
+ boolean amrmWeightsMatch = otherAMRMWeights != null &&
+ getAMRMPolicyWeights() != null &&
+ CollectionUtils.isEqualCollection(otherAMRMWeights.entrySet(),
+ getAMRMPolicyWeights().entrySet());
+
+ boolean routerWeightsMatch = otherRouterWeights != null &&
+ getRouterPolicyWeights() != null &&
+ CollectionUtils.isEqualCollection(otherRouterWeights.entrySet(),
+ getRouterPolicyWeights().entrySet());
+
+ return amrmWeightsMatch && routerWeightsMatch;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * amrmPolicyWeights.hashCode() + routerPolicyWeights.hashCode();
+ }
+
+ /**
+ * Return the parameter headroomAlpha, used by policies that balance
+ * weight-based and load-based considerations in their decisions.
+ *
+ * For policies that use this parameter, values close to 1 indicate that
+ * most of the decision should be based on currently observed headroom from
+ * various sub-clusters, values close to zero, indicate that the decision
+ * should be mostly based on weights and practically ignore current load.
+ *
+ * @return the value of headroomAlpha.
+ */
+ public float getHeadroomAlpha() {
+ return headroomAlpha;
+ }
+
+ /**
+ * Set the parameter headroomAlpha, used by policies that balance
+ * weight-based and load-based considerations in their decisions.
+ *
+ * For policies that use this parameter, values close to 1 indicate that
+ * most of the decision should be based on currently observed headroom from
+ * various sub-clusters, values close to zero, indicate that the decision
+ * should be mostly based on weights and practically ignore current load.
+ *
+ * @param headroomAlpha the value to use for balancing.
+ */
+ public void setHeadroomAlpha(float headroomAlpha) {
+ this.headroomAlpha = headroomAlpha;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return toJSONString();
+ } catch (JAXBException e) {
+ e.printStackTrace();
+ return "Error serializing to string.";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
new file mode 100644
index 0000000..43f5b83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** DAO objects for serializing/deserializing policy configurations. **/
+package org.apache.hadoop.yarn.server.federation.policies.dao;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
new file mode 100644
index 0000000..e888979
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.Map;
+
+/**
+ * Abstract class provides common validation of reinitialize(), for all
+ * policies that are "weight-based".
+ */
+public abstract class BaseWeightedRouterPolicy
+ implements FederationRouterPolicy {
+
+ private WeightedPolicyInfo policyInfo = null;
+ private FederationPolicyInitializationContext policyContext;
+
+ public BaseWeightedRouterPolicy() {
+ }
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext
+ federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ // perform consistency checks
+ WeightedPolicyInfo newPolicyInfo = WeightedPolicyInfo
+ .fromByteBuffer(
+ federationPolicyContext.getSubClusterPolicyConfiguration()
+ .getParams());
+
+ // if nothing has changed skip the rest of initialization
+ if (policyInfo != null && policyInfo.equals(newPolicyInfo)) {
+ return;
+ }
+
+ validate(newPolicyInfo);
+ setPolicyInfo(newPolicyInfo);
+ this.policyContext = federationPolicyContext;
+ }
+
+ /**
+ * Overridable validation step for the policy configuration.
+ * @param newPolicyInfo the configuration to test.
+ * @throws FederationPolicyInitializationException if the configuration is
+ * not valid.
+ */
+ public void validate(WeightedPolicyInfo newPolicyInfo) throws
+ FederationPolicyInitializationException {
+ if (newPolicyInfo == null) {
+ throw new FederationPolicyInitializationException("The policy to "
+ + "validate should not be null.");
+ }
+ Map<SubClusterIdInfo, Float> newWeights =
+ newPolicyInfo.getRouterPolicyWeights();
+ if (newWeights == null || newWeights.size() < 1) {
+ throw new FederationPolicyInitializationException(
+ "Weight vector cannot be null/empty.");
+ }
+ }
+
+
+ /**
+ * Getter method for the configuration weights.
+ *
+ * @return the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public WeightedPolicyInfo getPolicyInfo() {
+ return policyInfo;
+ }
+
+ /**
+ * Setter method for the configuration weights.
+ *
+ * @param policyInfo the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public void setPolicyInfo(
+ WeightedPolicyInfo policyInfo) {
+ this.policyInfo = policyInfo;
+ }
+
+ /**
+ * Getter method for the {@link FederationPolicyInitializationContext}.
+ * @return the context for this policy.
+ */
+ public FederationPolicyInitializationContext getPolicyContext() {
+ return policyContext;
+ }
+
+ /**
+ * Setter method for the {@link FederationPolicyInitializationContext}.
+ * @param policyContext the context to assign to this policy.
+ */
+ public void setPolicyContext(
+ FederationPolicyInitializationContext policyContext) {
+ this.policyContext = policyContext;
+ }
+
+ /**
+ * This methods gets active subclusters map from the {@code
+ * FederationStateStoreFacade} and validate it not being null/empty.
+ *
+ * @return the map of ids to info for all active subclusters.
+ * @throws YarnException if we can't get the list.
+ */
+ protected Map<SubClusterId, SubClusterInfo> getActiveSubclusters()
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters = getPolicyContext()
+ .getFederationStateStoreFacade().getSubClusters(true);
+
+ if (activeSubclusters == null || activeSubclusters.size() < 1) {
+ throw new NoActiveSubclustersException(
+ "Zero active subclusters, cannot pick where to send job.");
+ }
+ return activeSubclusters;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
new file mode 100644
index 0000000..e57709f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.Map;
+
+/**
+ * This implements a simple load-balancing policy. The policy "weights" are
+ * binary 0/1 values that enable/disable each sub-cluster, and the policy peaks
+ * the sub-cluster with the least load to forward this application.
+ */
+public class LoadBasedRouterPolicy
+ extends BaseWeightedRouterPolicy {
+
+ private static final Log LOG =
+ LogFactory.getLog(LoadBasedRouterPolicy.class);
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext
+ federationPolicyContext)
+ throws FederationPolicyInitializationException {
+
+ // remember old policyInfo
+ WeightedPolicyInfo tempPolicy = getPolicyInfo();
+
+ //attempt new initialization
+ super.reinitialize(federationPolicyContext);
+
+ //check extra constraints
+ for (Float weight : getPolicyInfo().getRouterPolicyWeights().values()) {
+ if (weight != 0 && weight != 1) {
+ //reset to old policyInfo if check fails
+ setPolicyInfo(tempPolicy);
+ throw new FederationPolicyInitializationException(
+ this.getClass().getCanonicalName()
+ + " policy expects all weights to be either "
+ + "\"0\" or \"1\"");
+ }
+ }
+ }
+
+ @Override
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
+ .getRouterPolicyWeights();
+ SubClusterIdInfo chosen = null;
+ long currBestMem = -1;
+ for (Map.Entry<SubClusterId, SubClusterInfo> entry :
+ activeSubclusters
+ .entrySet()) {
+ SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey());
+ if (weights.containsKey(id) && weights.get(id) > 0) {
+ long availableMemory = getAvailableMemory(entry.getValue());
+ if (availableMemory > currBestMem) {
+ currBestMem = availableMemory;
+ chosen = id;
+ }
+ }
+ }
+
+ return chosen.toId();
+ }
+
+ private long getAvailableMemory(SubClusterInfo value)
+ throws YarnException {
+ try {
+ long mem = -1;
+ JSONObject obj = new JSONObject(value.getCapability());
+ mem = obj.getJSONObject("clusterMetrics").getLong("availableMB");
+ return mem;
+ } catch (JSONException j) {
+ throw new YarnException("FederationSubCluserInfo cannot be parsed", j);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
new file mode 100644
index 0000000..a8ac5f7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.Map;
+
+/**
+ * This implements a policy that interprets "weights" as a ordered list of
+ * preferences among sub-clusters. Highest weight among active subclusters is
+ * chosen.
+ */
+public class PriorityRouterPolicy
+ extends BaseWeightedRouterPolicy {
+
+ private static final Log LOG =
+ LogFactory.getLog(PriorityRouterPolicy.class);
+
+ @Override
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ // This finds the sub-cluster with the highest weight among the
+ // currently active ones.
+ Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
+ .getRouterPolicyWeights();
+ SubClusterId chosen = null;
+ Float currentBest = Float.MIN_VALUE;
+ for (SubClusterId id : activeSubclusters.keySet()) {
+ SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
+ if (weights.containsKey(idInfo) && weights.get(idInfo) > currentBest) {
+ currentBest = weights.get(idInfo);
+ chosen = id;
+ }
+ }
+
+ return chosen;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
new file mode 100644
index 0000000..1774961
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * This simple policy picks at uniform random among any of the currently active
+ * subclusters. This policy is easy to use and good for testing.
+ *
+ * NOTE: this is "almost" subsumed by the {@code WeightedRandomRouterPolicy}.
+ * Behavior only diverges when there are active sub-clusters that are not part
+ * of the "weights", in which case the {@link UniformRandomRouterPolicy} send
+ * load to them, while {@code WeightedRandomRouterPolicy} does not.
+ */
+public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
+
+ private Random rand;
+
+ public UniformRandomRouterPolicy() {
+ rand = new Random(System.currentTimeMillis());
+ }
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ //note: this overrides BaseWeighterRouterPolicy and ignores the weights
+
+ setPolicyContext(federationPolicyContext);
+ }
+
+ /**
+ * Simply picks a random active subcluster to start the AM (this does NOT
+ * depend on the weights in the policy).
+ *
+ * @param appSubmissionContext the context for the app being submitted
+ * (ignored).
+ *
+ * @return a randomly chosen subcluster.
+ *
+ * @throws YarnException if there are no active subclusters.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ List<SubClusterId> list =
+ new ArrayList<>(activeSubclusters.keySet());
+ return list.get(rand.nextInt(list.size()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
new file mode 100644
index 0000000..0777677
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * This policy implements a weighted random sample among currently active
+ * sub-clusters.
+ */
+public class WeightedRandomRouterPolicy
+ extends BaseWeightedRouterPolicy {
+
+ private static final Log LOG =
+ LogFactory.getLog(WeightedRandomRouterPolicy.class);
+ private Random rand = new Random(System.currentTimeMillis());
+
+ @Override
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ // note: we cannot pre-compute the weights, as the set of activeSubcluster
+ // changes dynamically (and this would unfairly spread the load to
+ // sub-clusters adjacent to an inactive one), hence we need to count/scan
+ // the list and based on weight pick the next sub-cluster.
+ Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
+ .getRouterPolicyWeights();
+
+ float totActiveWeight = 0;
+ for(Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()){
+ if(entry.getKey()!=null && activeSubclusters.containsKey(entry.getKey()
+ .toId())){
+ totActiveWeight += entry.getValue();
+ }
+ }
+ float lookupValue = rand.nextFloat() * totActiveWeight;
+
+ for (SubClusterId id : activeSubclusters.keySet()) {
+ SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
+ if (weights.containsKey(idInfo)) {
+ lookupValue -= weights.get(idInfo);
+ }
+ if (lookupValue <= 0) {
+ return id;
+ }
+ }
+ //should never happen
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
new file mode 100644
index 0000000..e2260a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class represent a sub-cluster identifier in the JSON representation
+ * of the policy configuration.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@XmlRootElement(name = "federation-policy")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SubClusterIdInfo {
+
+ private String id;
+
+ public SubClusterIdInfo() {
+ //JAXB needs this
+ }
+
+ public SubClusterIdInfo(String subClusterId) {
+ this.id = subClusterId;
+ }
+
+ public SubClusterIdInfo(SubClusterId subClusterId) {
+ this.id = subClusterId.getId();
+ }
+
+ /**
+ * Get the sub-cluster identifier as {@link SubClusterId}.
+ * @return the sub-cluster id.
+ */
+ public SubClusterId toId() {
+ return SubClusterId.newInstance(id);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof SubClusterIdInfo) {
+ if (((SubClusterIdInfo) other).id.equals(this.id)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
new file mode 100644
index 0000000..8da92b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import static org.mockito.Mockito.mock;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Test;
+
+/**
+ * Base class for policies tests, tests for common reinitialization cases.
+ */
+public abstract class BaseFederationPoliciesTest {
+
+ private ConfigurableFederationPolicy policy;
+ private WeightedPolicyInfo policyInfo;
+ private Map<SubClusterId, SubClusterInfo> activeSubclusters = new HashMap<>();
+ private FederationPolicyInitializationContext federationPolicyContext;
+ private ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ private Random rand = new Random();
+
+ @Test
+ public void testReinitilialize() throws YarnException {
+ FederationPolicyInitializationContext fpc =
+ new FederationPolicyInitializationContext();
+ ByteBuffer buf = getPolicyInfo().toByteBuffer();
+ fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+ .newInstance("queue1", getPolicy().getClass().getCanonicalName(), buf));
+ fpc.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ getPolicy().reinitialize(fpc);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testReinitilializeBad1() throws YarnException {
+ getPolicy().reinitialize(null);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testReinitilializeBad2() throws YarnException {
+ FederationPolicyInitializationContext fpc =
+ new FederationPolicyInitializationContext();
+ getPolicy().reinitialize(fpc);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testReinitilializeBad3() throws YarnException {
+ FederationPolicyInitializationContext fpc =
+ new FederationPolicyInitializationContext();
+ ByteBuffer buf = mock(ByteBuffer.class);
+ fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+ .newInstance("queue1", "WrongPolicyName", buf));
+ fpc.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ getPolicy().reinitialize(fpc);
+ }
+
+ @Test(expected = NoActiveSubclustersException.class)
+ public void testNoSubclusters() throws YarnException {
+ // empty the activeSubclusters map
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), new HashMap<>());
+
+ ConfigurableFederationPolicy currentPolicy = getPolicy();
+ if (currentPolicy instanceof FederationRouterPolicy) {
+ ((FederationRouterPolicy) currentPolicy)
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ }
+ }
+
+ public ConfigurableFederationPolicy getPolicy() {
+ return policy;
+ }
+
+ public void setPolicy(ConfigurableFederationPolicy policy) {
+ this.policy = policy;
+ }
+
+ public WeightedPolicyInfo getPolicyInfo() {
+ return policyInfo;
+ }
+
+ public void setPolicyInfo(WeightedPolicyInfo policyInfo) {
+ this.policyInfo = policyInfo;
+ }
+
+ public Map<SubClusterId, SubClusterInfo> getActiveSubclusters() {
+ return activeSubclusters;
+ }
+
+ public void setActiveSubclusters(
+ Map<SubClusterId, SubClusterInfo> activeSubclusters) {
+ this.activeSubclusters = activeSubclusters;
+ }
+
+ public FederationPolicyInitializationContext getFederationPolicyContext() {
+ return federationPolicyContext;
+ }
+
+ public void setFederationPolicyContext(
+ FederationPolicyInitializationContext federationPolicyContext) {
+ this.federationPolicyContext = federationPolicyContext;
+ }
+
+ public ApplicationSubmissionContext getApplicationSubmissionContext() {
+ return applicationSubmissionContext;
+ }
+
+ public void setApplicationSubmissionContext(
+ ApplicationSubmissionContext applicationSubmissionContext) {
+ this.applicationSubmissionContext = applicationSubmissionContext;
+ }
+
+ public Random getRand() {
+ return rand;
+ }
+
+ public void setRand(Random rand) {
+ this.rand = rand;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index 4ec04d5..e840b3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -77,7 +77,7 @@ public class TestFederationPolicyInitializationContextValidator {
@Test(expected = FederationPolicyInitializationException.class)
public void nullConf() throws Exception {
- context.setFederationPolicyConfiguration(null);
+ context.setSubClusterPolicyConfiguration(null);
FederationPolicyInitializationContextValidator.validate(context,
MockPolicyManager.class.getCanonicalName());
}
@@ -96,8 +96,8 @@ public class TestFederationPolicyInitializationContextValidator {
MockPolicyManager.class.getCanonicalName());
}
- private class MockPolicyManager
- implements FederationPolicyWriter, FederationPolicyConfigurator {
+ private class MockPolicyManager implements FederationPolicyManager {
+
@Override
public FederationAMRMProxyPolicy getAMRMPolicy(
FederationPolicyInitializationContext
@@ -123,6 +123,17 @@ public class TestFederationPolicyInitializationContextValidator {
return SubClusterPolicyConfiguration
.newInstance("queue1", this.getClass().getCanonicalName(), buf);
}
+
+ @Override
+ public String getQueue() {
+ return "default";
+ }
+
+ @Override
+ public void setQueue(String queue) {
+
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
new file mode 100644
index 0000000..9e94f72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the
+ * load is properly considered for allocation.
+ */
+public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new LoadBasedRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 active subclusters
+ for (int i = 0; i < 20; i++) {
+ SubClusterIdInfo sc =
+ new SubClusterIdInfo(String.format("sc%02d", i));
+ SubClusterInfo federationSubClusterInfo =
+ SubClusterInfo.newInstance(sc.toId(), null, null, null, null, -1,
+ SubClusterState.SC_RUNNING, -1,
+ generateClusterMetricsInfo(i));
+ getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
+ float weight = getRand().nextInt(2);
+ if (i == 5) {
+ weight = 1.0f;
+ }
+
+ // 5% chance we omit one of the weights
+ if (i <= 5 || getRand().nextFloat() > 0.05f) {
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ }
+ }
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+
+ }
+
+ private String generateClusterMetricsInfo(int id) {
+
+ long mem = 1024 * getRand().nextInt(277 * 100 - 1);
+ //plant a best cluster
+ if (id == 5) {
+ mem = 1024 * 277 * 100;
+ }
+ String clusterMetrics =
+ "{\"clusterMetrics\":{\"appsSubmitted\":65," + "\"appsCompleted\":64,"
+ + "\"appsPending\":0,\"appsRunning\":0,\"appsFailed\":0,"
+ + "\"appsKilled\":1,\"reservedMB\":0,\"availableMB\":" + mem + ","
+ + "\"allocatedMB\":0,\"reservedVirtualCores\":0,"
+ + "\"availableVirtualCores\":2216,\"allocatedVirtualCores\":0,"
+ + "\"containersAllocated\":0,\"containersReserved\":0,"
+ + "\"containersPending\":0,\"totalMB\":28364800,"
+ + "\"totalVirtualCores\":2216,\"totalNodes\":278,\"lostNodes\":1,"
+ + "\"unhealthyNodes\":0,\"decommissionedNodes\":0,"
+ + "\"rebootedNodes\":0,\"activeNodes\":277}}\n";
+
+ return clusterMetrics;
+
+ }
+
+ @Test
+ public void testLoadIsRespected() throws YarnException {
+
+ SubClusterId chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+
+ // check the "planted" best cluster is chosen
+ Assert.assertEquals("sc05", chosen.getId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
new file mode 100644
index 0000000..ff5175d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Simple test class for the {@link PriorityRouterPolicy}. Tests that the
+ * weights are correctly used for ordering the choice of sub-clusters.
+ */
+public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new PriorityRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 subclusters with a 5% chance of being inactive
+ for (int i = 0; i < 20; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+
+ // with 5% omit a subcluster
+ if (getRand().nextFloat() < 0.95f || i == 5) {
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+ float weight = getRand().nextFloat();
+ if (i == 5) {
+ weight = 1.1f; // guaranteed to be the largest.
+ }
+
+ // 5% chance we omit one of the weights
+ if (i <= 5 || getRand().nextFloat() > 0.05f) {
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ }
+ }
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(),
+ getActiveSubclusters());
+
+ }
+
+ @Test
+ public void testPickLowestWeight() throws YarnException {
+ SubClusterId chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ Assert.assertEquals("sc5", chosen.getId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
new file mode 100644
index 0000000..ac41ab5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link UniformRandomRouterPolicy}. Tests that one
+ * of the active subcluster is chosen.
+ */
+public class TestUniformRandomRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new UniformRandomRouterPolicy());
+ // needed for base test to work
+ setPolicyInfo(mock(WeightedPolicyInfo.class));
+ for (int i = 1; i <= 2; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ mock(WeightedPolicyInfo.class), getActiveSubclusters());
+ }
+
+ @Test
+ public void testOneSubclusterIsChosen() throws YarnException {
+ SubClusterId chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ Assert.assertTrue(getActiveSubclusters().keySet().contains(chosen));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
new file mode 100644
index 0000000..a612685
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Simple test class for the {@link WeightedRandomRouterPolicy}. Generate large
+ * number of randomized tests to check we are weighiting correctly even if
+ * clusters go inactive.
+ */
+public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new WeightedRandomRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 subclusters with a 5% chance of being inactive
+ for (int i = 0; i < 20; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ // with 5% omit a subcluster
+ if (getRand().nextFloat() < 0.95f) {
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+ // 5% chance we omit one of the weights
+ float weight = getRand().nextFloat();
+ if (i <= 5 || getRand().nextFloat() > 0.05f) {
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ }
+ }
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(),
+ getActiveSubclusters());
+
+ }
+
+ @Test
+ public void testClusterChosenWithRightProbability() throws YarnException {
+
+ Map<SubClusterId, AtomicLong> counter = new HashMap<>();
+ for (SubClusterIdInfo id : getPolicyInfo().getRouterPolicyWeights()
+ .keySet()) {
+ counter.put(id.toId(), new AtomicLong(0));
+ }
+
+ float numberOfDraws = 1000000;
+
+ for (float i = 0; i < numberOfDraws; i++) {
+ SubClusterId chosenId = ((FederationRouterPolicy) getPolicy()).
+ getHomeSubcluster(getApplicationSubmissionContext());
+ counter.get(chosenId).incrementAndGet();
+ }
+
+ float totalActiveWeight = 0;
+ for (SubClusterId id : getActiveSubclusters().keySet()) {
+ SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
+ if (getPolicyInfo().getRouterPolicyWeights().containsKey(idInfo)) {
+ totalActiveWeight +=
+ getPolicyInfo().getRouterPolicyWeights().get(idInfo);
+ }
+ }
+
+ for (Map.Entry<SubClusterId, AtomicLong> counterEntry : counter
+ .entrySet()) {
+ float expectedWeight = getPolicyInfo().getRouterPolicyWeights()
+ .get(new SubClusterIdInfo(counterEntry.getKey())) / totalActiveWeight;
+ float actualWeight = counterEntry.getValue().floatValue() / numberOfDraws;
+
+ // make sure that the weights is respected among active subclusters
+ // and no jobs are routed to inactive subclusters.
+ if (getActiveSubclusters().containsKey(counterEntry.getKey())) {
+ Assert.assertTrue(
+ "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ + " expected weight: " + expectedWeight, expectedWeight == 0 ||
+ (actualWeight / expectedWeight) < 1.1
+ && (actualWeight / expectedWeight) > 0.9);
+ } else {
+ Assert.assertTrue(
+ "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ + " expected weight: " + expectedWeight, actualWeight == 0);
+
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-13075. Add support for
SSE-KMS and SSE-C in s3a filesystem. (Steve Moist via lei)
Posted by su...@apache.org.
HADOOP-13075. Add support for SSE-KMS and SSE-C in s3a filesystem. (Steve Moist via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/839b690e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/839b690e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/839b690e
Branch: refs/heads/YARN-2915
Commit: 839b690ed5edc2ac4984640d58c005bb63cd8a07
Parents: 649deb7
Author: Lei Xu <le...@apache.org>
Authored: Sat Feb 11 13:59:03 2017 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Sat Feb 11 13:59:03 2017 -0800
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/pom.xml | 2 +
.../org/apache/hadoop/fs/s3a/Constants.java | 22 ++-
.../hadoop/fs/s3a/S3ABlockOutputStream.java | 1 +
.../hadoop/fs/s3a/S3AEncryptionMethods.java | 61 ++++++++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 131 ++++++++++++++--
.../apache/hadoop/fs/s3a/S3AInputStream.java | 24 ++-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 10 ++
.../hadoop/fs/s3a/S3ObjectAttributes.java | 59 +++++++
.../src/site/markdown/tools/hadoop-aws/index.md | 29 +++-
.../fs/s3a/AbstractTestS3AEncryption.java | 132 ++++++++++++++++
.../hadoop/fs/s3a/ITestS3AEncryption.java | 97 ------------
.../ITestS3AEncryptionAlgorithmPropagation.java | 76 ----------
.../ITestS3AEncryptionAlgorithmValidation.java | 152 +++++++++++++++++++
.../ITestS3AEncryptionBlockOutputStream.java | 36 -----
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 90 +++++++++++
...ITestS3AEncryptionSSECBlockOutputStream.java | 46 ++++++
.../s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 57 +++++++
.../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 48 ++++++
...onSSEKMSUserDefinedKeyBlockOutputStream.java | 52 +++++++
.../hadoop/fs/s3a/ITestS3AEncryptionSSES3.java | 43 ++++++
...TestS3AEncryptionSSES3BlockOutputStream.java | 45 ++++++
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 57 +++++--
22 files changed, 1030 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 6fd503c..1f64b02 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -183,6 +183,7 @@
<exclude>**/ITestJets3tNativeS3FileSystemContract.java</exclude>
<exclude>**/ITest*Root*.java</exclude>
<exclude>**/ITestS3AFileContextStatistics.java</exclude>
+ <exclude>**/ITestS3AEncryptionSSE*.java</exclude>
<include>**/ITestS3AHuge*.java</include>
</excludes>
</configuration>
@@ -211,6 +212,7 @@
<include>**/ITest*Root*.java</include>
<include>**/ITestS3AFileContextStatistics.java</include>
<include>**/ITestS3AHuge*.java</include>
+ <include>**/ITestS3AEncryptionSSE*.java</include>
</includes>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 45d974c..414f951 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -216,17 +216,28 @@ public final class Constants {
"fs.s3a.multipart.purge.age";
public static final long DEFAULT_PURGE_EXISTING_MULTIPART_AGE = 86400;
- // s3 server-side encryption
+ // s3 server-side encryption, see S3AEncryptionMethods for valid options
public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM =
"fs.s3a.server-side-encryption-algorithm";
/**
* The standard encryption algorithm AWS supports.
* Different implementations may support others (or none).
+ * Use the S3AEncryptionMethods instead when configuring
+ * which Server Side Encryption to use.
*/
+ @Deprecated
public static final String SERVER_SIDE_ENCRYPTION_AES256 =
"AES256";
+ /**
+ * Used to specify which AWS KMS key to use if
+ * SERVER_SIDE_ENCRYPTION_ALGORITHM is AWS_KMS (will default to aws/s3
+ * master key if left blank) or with SSE_C, the actual AES 256 key.
+ */
+ public static final String SERVER_SIDE_ENCRYPTION_KEY =
+ "fs.s3a.server-side-encryption-key";
+
//override signature algorithm used for signing requests
public static final String SIGNING_ALGORITHM = "fs.s3a.signing-algorithm";
@@ -296,4 +307,13 @@ public final class Constants {
*/
@InterfaceAudience.Private
public static final int MAX_MULTIPART_COUNT = 10000;
+
+ @InterfaceAudience.Private
+ public static final String SSE_C_NO_KEY_ERROR = S3AEncryptionMethods.SSE_C
+ .getMethod() +" is enabled and no encryption key is provided.";
+
+
+ @InterfaceAudience.Private
+ public static final String SSE_S3_WITH_KEY_ERROR = S3AEncryptionMethods.SSE_S3
+ .getMethod() +" is configured and an " + "encryption key is provided";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index b66a23f..89b9b29 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -382,6 +382,7 @@ class S3ABlockOutputStream extends OutputStream {
writeOperationHelper.newPutRequest(
block.startUpload(),
size);
+ fs.setOptionalPutRequestParameters(putObjectRequest);
long transferQueueTime = now();
BlockUploadProgress callback =
new BlockUploadProgress(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java
new file mode 100644
index 0000000..1b3d86c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * This enum is to centralize the encryption methods and
+ * the value required in the configuration.
+ */
+public enum S3AEncryptionMethods {
+
+ SSE_S3("AES256"),
+ SSE_KMS("SSE-KMS"),
+ SSE_C("SSE-C"),
+ NONE("");
+
+ private String method;
+
+ S3AEncryptionMethods(String method) {
+ this.method = method;
+ }
+
+ public String getMethod() {
+ return method;
+ }
+
+ public static S3AEncryptionMethods getMethod(String name) throws IOException {
+ if(StringUtils.isBlank(name)) {
+ return NONE;
+ }
+ switch(name) {
+ case "AES256":
+ return SSE_S3;
+ case "SSE-KMS":
+ return SSE_KMS;
+ case "SSE-C":
+ return SSE_C;
+ default:
+ throw new IOException("Unknown Server Side algorithm "+name);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 8152bf8..bffc210 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -43,6 +43,7 @@ import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
@@ -51,6 +52,8 @@ import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
+import com.amazonaws.services.s3.model.SSECustomerKey;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.amazonaws.services.s3.transfer.Copy;
@@ -135,7 +138,7 @@ public class S3AFileSystem extends FileSystem {
LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
private LocalDirAllocator directoryAllocator;
private CannedAccessControlList cannedACL;
- private String serverSideEncryptionAlgorithm;
+ private S3AEncryptionMethods serverSideEncryptionAlgorithm;
private S3AInstrumentation instrumentation;
private S3AStorageStatistics storageStatistics;
private long readAhead;
@@ -227,8 +230,17 @@ public class S3AFileSystem extends FileSystem {
initMultipartUploads(conf);
- serverSideEncryptionAlgorithm =
- conf.getTrimmed(SERVER_SIDE_ENCRYPTION_ALGORITHM);
+ serverSideEncryptionAlgorithm = S3AEncryptionMethods.getMethod(
+ conf.getTrimmed(SERVER_SIDE_ENCRYPTION_ALGORITHM));
+ if(S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) &&
+ StringUtils.isBlank(getServerSideEncryptionKey(getConf()))) {
+ throw new IOException(Constants.SSE_C_NO_KEY_ERROR);
+ }
+ if(S3AEncryptionMethods.SSE_S3.equals(serverSideEncryptionAlgorithm) &&
+ StringUtils.isNotBlank(getServerSideEncryptionKey(
+ getConf()))) {
+ throw new IOException(Constants.SSE_S3_WITH_KEY_ERROR);
+ }
LOG.debug("Using encryption {}", serverSideEncryptionAlgorithm);
inputPolicy = S3AInputPolicy.getPolicy(
conf.getTrimmed(INPUT_FADVISE, INPUT_FADV_NORMAL));
@@ -514,9 +526,18 @@ public class S3AFileSystem extends FileSystem {
+ " because it is a directory");
}
- return new FSDataInputStream(new S3AInputStream(bucket, pathToKey(f),
- fileStatus.getLen(), s3, statistics, instrumentation, readAhead,
- inputPolicy));
+ return new FSDataInputStream(
+ new S3AInputStream(new S3ObjectAttributes(
+ bucket,
+ pathToKey(f),
+ serverSideEncryptionAlgorithm,
+ getServerSideEncryptionKey(getConf())),
+ fileStatus.getLen(),
+ s3,
+ statistics,
+ instrumentation,
+ readAhead,
+ inputPolicy));
}
/**
@@ -892,7 +913,14 @@ public class S3AFileSystem extends FileSystem {
*/
protected ObjectMetadata getObjectMetadata(String key) {
incrementStatistic(OBJECT_METADATA_REQUESTS);
- ObjectMetadata meta = s3.getObjectMetadata(bucket, key);
+ GetObjectMetadataRequest request =
+ new GetObjectMetadataRequest(bucket, key);
+ //SSE-C requires to be filled in if enabled for object metadata
+ if(S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) &&
+ StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))){
+ request.setSSECustomerKey(generateSSECustomerKey());
+ }
+ ObjectMetadata meta = s3.getObjectMetadata(request);
incrementReadOperations();
return meta;
}
@@ -986,6 +1014,7 @@ public class S3AFileSystem extends FileSystem {
ObjectMetadata metadata, File srcfile) {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key,
srcfile);
+ setOptionalPutRequestParameters(putObjectRequest);
putObjectRequest.setCannedAcl(cannedACL);
putObjectRequest.setMetadata(metadata);
return putObjectRequest;
@@ -1004,6 +1033,7 @@ public class S3AFileSystem extends FileSystem {
ObjectMetadata metadata, InputStream inputStream) {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key,
inputStream, metadata);
+ setOptionalPutRequestParameters(putObjectRequest);
putObjectRequest.setCannedAcl(cannedACL);
return putObjectRequest;
}
@@ -1016,9 +1046,7 @@ public class S3AFileSystem extends FileSystem {
*/
public ObjectMetadata newObjectMetadata() {
final ObjectMetadata om = new ObjectMetadata();
- if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
- om.setSSEAlgorithm(serverSideEncryptionAlgorithm);
- }
+ setOptionalObjectMetadata(om);
return om;
}
@@ -1752,11 +1780,10 @@ public class S3AFileSystem extends FileSystem {
try {
ObjectMetadata srcom = getObjectMetadata(srcKey);
ObjectMetadata dstom = cloneObjectMetadata(srcom);
- if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
- dstom.setSSEAlgorithm(serverSideEncryptionAlgorithm);
- }
+ setOptionalObjectMetadata(dstom);
CopyObjectRequest copyObjectRequest =
new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
+ setOptionalCopyObjectRequestParameters(copyObjectRequest);
copyObjectRequest.setCannedAccessControlList(cannedACL);
copyObjectRequest.setNewObjectMetadata(dstom);
@@ -1788,6 +1815,83 @@ public class S3AFileSystem extends FileSystem {
}
}
+ protected void setOptionalMultipartUploadRequestParameters(
+ InitiateMultipartUploadRequest req) {
+ switch (serverSideEncryptionAlgorithm) {
+ case SSE_KMS:
+ req.setSSEAwsKeyManagementParams(generateSSEAwsKeyParams());
+ break;
+ case SSE_C:
+ if (StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))) {
+ //at the moment, only supports copy using the same key
+ req.setSSECustomerKey(generateSSECustomerKey());
+ }
+ break;
+ default:
+ }
+ }
+
+
+ protected void setOptionalCopyObjectRequestParameters(
+ CopyObjectRequest copyObjectRequest) throws IOException {
+ switch (serverSideEncryptionAlgorithm) {
+ case SSE_KMS:
+ copyObjectRequest.setSSEAwsKeyManagementParams(
+ generateSSEAwsKeyParams()
+ );
+ break;
+ case SSE_C:
+ if (StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))) {
+ //at the moment, only supports copy using the same key
+ SSECustomerKey customerKey = generateSSECustomerKey();
+ copyObjectRequest.setSourceSSECustomerKey(customerKey);
+ copyObjectRequest.setDestinationSSECustomerKey(customerKey);
+ }
+ break;
+ default:
+ }
+ }
+
+ protected void setOptionalPutRequestParameters(PutObjectRequest request) {
+ switch (serverSideEncryptionAlgorithm) {
+ case SSE_KMS:
+ request.setSSEAwsKeyManagementParams(generateSSEAwsKeyParams());
+ break;
+ case SSE_C:
+ if (StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))) {
+ request.setSSECustomerKey(generateSSECustomerKey());
+ }
+ break;
+ default:
+ }
+ }
+
+ private void setOptionalObjectMetadata(ObjectMetadata metadata) {
+ if (S3AEncryptionMethods.SSE_S3.equals(serverSideEncryptionAlgorithm)) {
+ metadata.setSSEAlgorithm(serverSideEncryptionAlgorithm.getMethod());
+ }
+ }
+
+ private SSEAwsKeyManagementParams generateSSEAwsKeyParams() {
+ //Use specified key, otherwise default to default master aws/s3 key by AWS
+ SSEAwsKeyManagementParams sseAwsKeyManagementParams =
+ new SSEAwsKeyManagementParams();
+ if (StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))) {
+ sseAwsKeyManagementParams =
+ new SSEAwsKeyManagementParams(
+ getServerSideEncryptionKey(getConf())
+ );
+ }
+ return sseAwsKeyManagementParams;
+ }
+
+ private SSECustomerKey generateSSECustomerKey() {
+ SSECustomerKey customerKey = new SSECustomerKey(
+ getServerSideEncryptionKey(getConf())
+ );
+ return customerKey;
+ }
+
/**
* Perform post-write actions.
* @param key key written to
@@ -2240,6 +2344,7 @@ public class S3AFileSystem extends FileSystem {
key,
newObjectMetadata(-1));
initiateMPURequest.setCannedACL(cannedACL);
+ setOptionalMultipartUploadRequestParameters(initiateMPURequest);
try {
return s3.initiateMultipartUpload(initiateMPURequest)
.getUploadId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 3c4093d..7d322a5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -22,6 +22,7 @@ import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.s3.model.SSECustomerKey;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -36,6 +37,7 @@ import org.slf4j.Logger;
import java.io.EOFException;
import java.io.IOException;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
/**
@@ -78,6 +80,8 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
private final String uri;
public static final Logger LOG = S3AFileSystem.LOG;
private final S3AInstrumentation.InputStreamStatistics streamStatistics;
+ private S3AEncryptionMethods serverSideEncryptionAlgorithm;
+ private String serverSideEncryptionKey;
private final S3AInputPolicy inputPolicy;
private long readahead = Constants.DEFAULT_READAHEAD_RANGE;
@@ -98,24 +102,26 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
*/
private long contentRangeStart;
- public S3AInputStream(String bucket,
- String key,
+ public S3AInputStream(S3ObjectAttributes s3Attributes,
long contentLength,
AmazonS3 client,
FileSystem.Statistics stats,
S3AInstrumentation instrumentation,
long readahead,
S3AInputPolicy inputPolicy) {
- Preconditions.checkArgument(StringUtils.isNotEmpty(bucket), "No Bucket");
- Preconditions.checkArgument(StringUtils.isNotEmpty(key), "No Key");
- Preconditions.checkArgument(contentLength >= 0 , "Negative content length");
- this.bucket = bucket;
- this.key = key;
+ Preconditions.checkArgument(isNotEmpty(s3Attributes.getBucket()), "No Bucket");
+ Preconditions.checkArgument(isNotEmpty(s3Attributes.getKey()), "No Key");
+ Preconditions.checkArgument(contentLength >= 0, "Negative content length");
+ this.bucket = s3Attributes.getBucket();
+ this.key = s3Attributes.getKey();
this.contentLength = contentLength;
this.client = client;
this.stats = stats;
this.uri = "s3a://" + this.bucket + "/" + this.key;
this.streamStatistics = instrumentation.newInputStreamStatistics();
+ this.serverSideEncryptionAlgorithm =
+ s3Attributes.getServerSideEncryptionAlgorithm();
+ this.serverSideEncryptionKey = s3Attributes.getServerSideEncryptionKey();
this.inputPolicy = inputPolicy;
setReadahead(readahead);
}
@@ -145,6 +151,10 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
try {
GetObjectRequest request = new GetObjectRequest(bucket, key)
.withRange(targetPos, contentRangeFinish);
+ if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) &&
+ StringUtils.isNotBlank(serverSideEncryptionKey)){
+ request.setSSECustomerKey(new SSECustomerKey(serverSideEncryptionKey));
+ }
wrappedStream = client.getObject(request).getObjectContent();
contentRangeStart = targetPos;
if (wrappedStream == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index c4ff638..5311211 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -723,4 +723,14 @@ public final class S3AUtils {
"patch of " + S3A_SECURITY_CREDENTIAL_PROVIDER_PATH);
}
}
+
+ static String getServerSideEncryptionKey(Configuration conf) {
+ try {
+ return getPassword(conf, Constants.SERVER_SIDE_ENCRYPTION_KEY,
+ conf.getTrimmed(SERVER_SIDE_ENCRYPTION_KEY));
+ } catch (IOException e) {
+ LOG.error("Cannot retrieve SERVER_SIDE_ENCRYPTION_KEY", e);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
new file mode 100644
index 0000000..7c73a23
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+/**
+ * This class is only a holder for bucket, key, SSE Algorithm and SSE key
+ * attributes. It is only used in {@link S3AInputStream}
+ * as a way to reduce parameters being passed
+ * to the constructor of such class.
+ */
+class S3ObjectAttributes {
+ private String bucket;
+ private String key;
+ private S3AEncryptionMethods serverSideEncryptionAlgorithm;
+ private String serverSideEncryptionKey;
+
+ public S3ObjectAttributes(
+ String bucket,
+ String key,
+ S3AEncryptionMethods serverSideEncryptionAlgorithm,
+ String serverSideEncryptionKey) {
+ this.bucket = bucket;
+ this.key = key;
+ this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
+ this.serverSideEncryptionKey = serverSideEncryptionKey;
+ }
+
+ public String getBucket() {
+ return bucket;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public S3AEncryptionMethods getServerSideEncryptionAlgorithm() {
+ return serverSideEncryptionAlgorithm;
+ }
+
+ public String getServerSideEncryptionKey() {
+ return serverSideEncryptionKey;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index d804a59..2471a52 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -840,11 +840,21 @@ from placing its declaration on the command line.
<property>
<name>fs.s3a.server-side-encryption-algorithm</name>
<description>Specify a server-side encryption algorithm for s3a: file system.
- Unset by default, and the only other currently allowable value is AES256.
+ Unset by default. It supports the following values: 'AES256' (for SSE-S3), 'SSE-KMS'
+ and 'SSE-C'
</description>
</property>
<property>
+ <name>fs.s3a.server-side-encryption-key</name>
+ <description>Specific encryption key to use if fs.s3a.server-side-encryption-algorithm
+ has been set to 'SSE-KMS' or 'SSE-C'. In the case of SSE-C, the value of this property
+ should be the Base64 encoded key. If you are using SSE-KMS and leave this property empty,
+ you'll be using your default's S3 KMS key, otherwise you should set this property to
+ the specific KMS key id.</description>
+ </property>
+
+ <property>
<name>fs.s3a.buffer.dir</name>
<value>${hadoop.tmp.dir}/s3a</value>
<description>Comma separated list of directories that will be used to buffer file
@@ -2160,6 +2170,23 @@ that the file `contract-test-options.xml` does not contain any
secret credentials itself. As the auth keys XML file is kept out of the
source code tree, it is not going to get accidentally committed.
+### Configuring S3a Encryption
+
+For S3a encryption tests to run correctly, the
+`fs.s3a.server-side-encryption-key` must be configured in the s3a contract xml
+file with a AWS KMS encryption key arn as this value is different for each AWS
+KMS.
+
+Example:
+
+ <property>
+ <name>fs.s3a.server-side-encryption-key</name>
+ <value>arn:aws:kms:us-west-2:360379543683:key/071a86ff-8881-4ba0-9230-95af6d01ca01</value>
+ </property>
+
+You can also force all the tests to run with a specific SSE encryption method
+by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a
+contract file.
### Running the Tests
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
new file mode 100644
index 0000000..5150942
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.net.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
+/**
+ * Test whether or not encryption works by turning it on. Some checks
+ * are made for different file sizes as there have been reports that the
+ * file length may be rounded up to match word boundaries.
+ */
+public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ getSSEAlgorithm().getMethod());
+ return conf;
+ }
+
+ private static final int[] SIZES = {
+ 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 12 - 1
+ };
+
+ @Test
+ public void testEncryption() throws Throwable {
+ for (int size: SIZES) {
+ validateEncryptionForFilesize(size);
+ }
+ }
+
+ @Test
+ public void testEncryptionOverRename() throws Throwable {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ Path src = path(createFilename(1024));
+ byte[] data = dataset(1024, 'a', 'z');
+ S3AFileSystem fs = getFileSystem();
+ writeDataset(fs, src, data, data.length, 1024 * 1024, true);
+ ContractTestUtils.verifyFileContents(fs, src, data);
+ Path dest = path(src.getName() + "-copy");
+ fs.rename(src, dest);
+ ContractTestUtils.verifyFileContents(fs, dest, data);
+ assertEncrypted(dest);
+ }
+
+ protected void validateEncryptionForFilesize(int len) throws IOException {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ describe("Create an encrypted file of size " + len);
+ String src = createFilename(len);
+ Path path = writeThenReadFile(src, len);
+ assertEncrypted(path);
+ rm(getFileSystem(), path, false, false);
+ }
+
+ protected String createFilename(int len) {
+ return String.format("%s-%04x", methodName.getMethodName(), len);
+ }
+
+ /**
+ * Assert that at path references an encrypted blob.
+ * @param path path
+ * @throws IOException on a failure
+ */
+ protected void assertEncrypted(Path path) throws IOException {
+ ObjectMetadata md = getFileSystem().getObjectMetadata(path);
+ switch(getSSEAlgorithm()) {
+ case SSE_C:
+ assertEquals("AES256", md.getSSECustomerAlgorithm());
+ String md5Key = convertKeyToMd5();
+ assertEquals(md5Key, md.getSSECustomerKeyMd5());
+ break;
+ case SSE_KMS:
+ assertEquals("aws:kms", md.getSSEAlgorithm());
+ //S3 will return full arn of the key, so specify global arn in properties
+ assertEquals(this.getConfiguration().
+ getTrimmed(Constants.SERVER_SIDE_ENCRYPTION_KEY),
+ md.getSSEAwsKmsKeyId());
+ break;
+ default:
+ assertEquals("AES256", md.getSSEAlgorithm());
+ }
+ }
+
+ /**
+ * Decodes the SERVER_SIDE_ENCRYPTION_KEY from base64 into an AES key, then
+ * gets the md5 of it, then encodes it in base64 so it will match the version
+ * that AWS returns to us.
+ *
+ * @return md5'd base64 encoded representation of the server side encryption
+ * key
+ */
+ private String convertKeyToMd5() {
+ String base64Key = getConfiguration().getTrimmed(
+ Constants.SERVER_SIDE_ENCRYPTION_KEY
+ );
+ byte[] key = Base64.decodeBase64(base64Key);
+ byte[] md5 = DigestUtils.md5(key);
+ return Base64.encodeBase64String(md5).trim();
+ }
+
+ protected abstract S3AEncryptionMethods getSSEAlgorithm();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
deleted file mode 100644
index 8432789..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-
-/**
- * Test whether or not encryption works by turning it on. Some checks
- * are made for different file sizes as there have been reports that the
- * file length may be rounded up to match word boundaries.
- */
-public class ITestS3AEncryption extends AbstractS3ATestBase {
- private static final String AES256 = Constants.SERVER_SIDE_ENCRYPTION_AES256;
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
- AES256);
- return conf;
- }
-
- private static final int[] SIZES = {
- 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 12 - 1
- };
-
- @Test
- public void testEncryption() throws Throwable {
- for (int size: SIZES) {
- validateEncryptionForFilesize(size);
- }
- }
-
- @Test
- public void testEncryptionOverRename() throws Throwable {
- skipIfEncryptionTestsDisabled(getConfiguration());
- Path src = path(createFilename(1024));
- byte[] data = dataset(1024, 'a', 'z');
- S3AFileSystem fs = getFileSystem();
- writeDataset(fs, src, data, data.length, 1024 * 1024, true);
- ContractTestUtils.verifyFileContents(fs, src, data);
- Path dest = path(src.getName() + "-copy");
- fs.rename(src, dest);
- ContractTestUtils.verifyFileContents(fs, dest, data);
- assertEncrypted(dest);
- }
-
- protected void validateEncryptionForFilesize(int len) throws IOException {
- skipIfEncryptionTestsDisabled(getConfiguration());
- describe("Create an encrypted file of size " + len);
- String src = createFilename(len);
- Path path = writeThenReadFile(src, len);
- assertEncrypted(path);
- rm(getFileSystem(), path, false, false);
- }
-
- private String createFilename(int len) {
- return String.format("%s-%04x", methodName.getMethodName(), len);
- }
-
- /**
- * Assert that at path references an encrypted blob.
- * @param path path
- * @throws IOException on a failure
- */
- private void assertEncrypted(Path path) throws IOException {
- ObjectMetadata md = getFileSystem().getObjectMetadata(path);
- assertEquals(AES256, md.getSSEAlgorithm());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
deleted file mode 100644
index 96deb25..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-
-/**
- * Test whether or not encryption settings propagate by choosing an invalid
- * one. We expect the write to fail with a 400 bad request error
- */
-public class ITestS3AEncryptionAlgorithmPropagation
- extends AbstractS3ATestBase {
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
- "DES");
- return conf;
- }
-
- @Test
- public void testEncrypt0() throws Throwable {
- writeThenReadFileToFailure(0);
- }
-
- @Test
- public void testEncrypt256() throws Throwable {
- writeThenReadFileToFailure(256);
- }
-
- /**
- * Make this a no-op so test setup doesn't fail.
- * @param path path path
- * @throws IOException on any failure
- */
- @Override
- protected void mkdirs(Path path) throws IOException {
-
- }
-
- protected void writeThenReadFileToFailure(int len) throws IOException {
- skipIfEncryptionTestsDisabled(getConfiguration());
- describe("Create an encrypted file of size " + len);
- try {
- writeThenReadFile(methodName.getMethodName() + '-' + len, len);
- fail("Expected an exception about an illegal encryption algorithm");
- } catch (AWSS3IOException e) {
- assertStatusCode(e, 400);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmValidation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmValidation.java
new file mode 100644
index 0000000..0cd8ff4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmValidation.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Test whether or not encryption settings propagate by choosing an invalid
+ * one. We expect the S3AFileSystem to fail to initialize.
+ */
+@Ignore
+public class ITestS3AEncryptionAlgorithmValidation
+ extends AbstractS3ATestBase {
+
+ @Test
+ public void testEncryptionAlgorithmSetToDES() throws Throwable {
+ //skip tests if they aren't enabled
+ assumeEnabled();
+ intercept(IOException.class, "Unknown Server Side algorithm DES", () -> {
+
+ Configuration conf = super.createConfiguration();
+ //DES is an invalid encryption algorithm
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM, "DES");
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+ //extract the test FS
+ FileSystem fileSystem = contract.getTestFileSystem();
+ assertNotNull("null filesystem", fileSystem);
+ URI fsURI = fileSystem.getUri();
+ LOG.info("Test filesystem = {} implemented by {}", fsURI, fileSystem);
+ assertEquals("wrong filesystem of " + fsURI,
+ contract.getScheme(), fsURI.getScheme());
+ fileSystem.initialize(fsURI, conf);
+ throw new Exception("Do not reach here");
+ });
+ }
+
+ @Test
+ public void testEncryptionAlgorithmSSECWithNoEncryptionKey() throws
+ Throwable {
+ //skip tests if they aren't enabled
+ assumeEnabled();
+ intercept(IllegalArgumentException.class, "The value of property " +
+ Constants.SERVER_SIDE_ENCRYPTION_KEY + " must not be null", () -> {
+
+ Configuration conf = super.createConfiguration();
+ //SSE-C must be configured with an encryption key
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ S3AEncryptionMethods.SSE_C.getMethod());
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, null);
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+ //extract the test FS
+ FileSystem fileSystem = contract.getTestFileSystem();
+ assertNotNull("null filesystem", fileSystem);
+ URI fsURI = fileSystem.getUri();
+ LOG.info("Test filesystem = {} implemented by {}", fsURI, fileSystem);
+ assertEquals("wrong filesystem of " + fsURI,
+ contract.getScheme(), fsURI.getScheme());
+ fileSystem.initialize(fsURI, conf);
+ throw new Exception("Do not reach here");
+ });
+ }
+
+ @Test
+ public void testEncryptionAlgorithmSSECWithBlankEncryptionKey() throws
+ Throwable {
+ intercept(IOException.class, Constants.SSE_C_NO_KEY_ERROR, () -> {
+
+ Configuration conf = super.createConfiguration();
+ //SSE-C must be configured with an encryption key
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ S3AEncryptionMethods.SSE_C.getMethod());
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, "");
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+ //extract the test FS
+ FileSystem fileSystem = contract.getTestFileSystem();
+ assertNotNull("null filesystem", fileSystem);
+ URI fsURI = fileSystem.getUri();
+ LOG.info("Test filesystem = {} implemented by {}", fsURI, fileSystem);
+ assertEquals("wrong filesystem of " + fsURI,
+ contract.getScheme(), fsURI.getScheme());
+ fileSystem.initialize(fsURI, conf);
+ throw new Exception("Do not reach here");
+ });
+ }
+
+ @Test
+ public void testEncryptionAlgorithmSSES3WithEncryptionKey() throws
+ Throwable {
+ //skip tests if they aren't enabled
+ assumeEnabled();
+ intercept(IOException.class, Constants.SSE_S3_WITH_KEY_ERROR, () -> {
+
+ Configuration conf = super.createConfiguration();
+ //SSE-S3 cannot be configured with an encryption key
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ S3AEncryptionMethods.SSE_S3.getMethod());
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY,
+ "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=");
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+ //extract the test FS
+ FileSystem fileSystem = contract.getTestFileSystem();
+ assertNotNull("null filesystem", fileSystem);
+ URI fsURI = fileSystem.getUri();
+ LOG.info("Test filesystem = {} implemented by {}", fsURI, fileSystem);
+ assertEquals("wrong filesystem of " + fsURI,
+ contract.getScheme(), fsURI.getScheme());
+ fileSystem.initialize(fsURI, conf);
+ throw new Exception("Do not reach here");
+ });
+ }
+
+ /**
+ * Make this a no-op so test setup doesn't fail.
+ * @param path path path
+ * @throws IOException on any failure
+ */
+ @Override
+ protected void mkdirs(Path path) throws IOException {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionBlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionBlockOutputStream.java
deleted file mode 100644
index 5239f30..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionBlockOutputStream.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Run the encryption tests against the block output stream.
- */
-public class ITestS3AEncryptionBlockOutputStream extends ITestS3AEncryption {
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- conf.set(Constants.FAST_UPLOAD_BUFFER,
- Constants.FAST_UPLOAD_BYTEBUFFER);
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
new file mode 100644
index 0000000..71586b8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.rm;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Test;
+
+/**
+ * Concrete class that extends {@link AbstractTestS3AEncryption}
+ * and tests SSE-C encryption.
+ */
+public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ getSSEAlgorithm().getMethod());
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY,
+ "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=");
+ return conf;
+ }
+
+ /**
+ * This will create and write to a file using encryption key A, then attempt
+ * to read from it again with encryption key B. This will not work as it
+ * cannot decrypt the file.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateFileAndReadWithDifferentEncryptionKey() throws
+ Exception {
+ final Path[] path = new Path[1];
+ intercept(java.nio.file.AccessDeniedException.class,
+ "Forbidden (Service: Amazon S3; Status Code: 403;", () -> {
+
+ int len = 2048;
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ describe("Create an encrypted file of size " + len);
+ String src = createFilename(len);
+ path[0] = writeThenReadFile(src, len);
+
+ Configuration conf = this.createConfiguration();
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY,
+ "kX7SdwVc/1VXJr76kfKnkQ3ONYhxianyL2+C3rPVT9s=");
+
+ S3AContract contract = (S3AContract) createContract(conf);
+ contract.init();
+ //skip tests if they aren't enabled
+ assumeEnabled();
+ //extract the test FS
+ FileSystem fileSystem = contract.getTestFileSystem();
+ byte[] data = dataset(len, 'a', 'z');
+ ContractTestUtils.verifyFileContents(fileSystem, path[0], data);
+ throw new Exception("Fail");
+ });
+ rm(getFileSystem(), path[0], false, false);
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_C;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSECBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSECBlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSECBlockOutputStream.java
new file mode 100644
index 0000000..afa0441
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSECBlockOutputStream.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Run the encryption tests against the Fast output stream.
+ * This verifies that both file writing paths can encrypt their data.
+ */
+
+public class ITestS3AEncryptionSSECBlockOutputStream
+ extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ conf.set(Constants.FAST_UPLOAD_BUFFER,
+ Constants.FAST_UPLOAD_BYTEBUFFER);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY,
+ "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=");
+ return conf;
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_C;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java
new file mode 100644
index 0000000..8b68fcf
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.hamcrest.CoreMatchers.containsString;
+
+import java.io.IOException;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Concrete class that extends {@link AbstractTestS3AEncryption}
+ * and tests SSE-KMS encryption when no KMS encryption key is provided and AWS
+ * uses the default. Since this resource changes for every account and region,
+ * there is no good way to explicitly set this value to do a equality check
+ * in the response.
+ */
+public class ITestS3AEncryptionSSEKMSDefaultKey
+ extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, "");
+ return conf;
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_KMS;
+ }
+
+ @Override
+ protected void assertEncrypted(Path path) throws IOException {
+ ObjectMetadata md = getFileSystem().getObjectMetadata(path);
+ assertEquals("aws:kms", md.getSSEAlgorithm());
+ assertThat(md.getSSEAwsKmsKeyId(), containsString("arn:aws:kms:"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
new file mode 100644
index 0000000..50c9fb5
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Concrete class that extends {@link AbstractTestS3AEncryption}
+ * and tests SSE-KMS encryption. This requires the SERVER_SIDE_ENCRYPTION_KEY
+ * to be set in auth-keys.xml for it to run.
+ */
+public class ITestS3AEncryptionSSEKMSUserDefinedKey
+ extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ if(StringUtils.isBlank(conf.get(Constants.SERVER_SIDE_ENCRYPTION_KEY))){
+ skip(Constants.SERVER_SIDE_ENCRYPTION_KEY+ " is not set for " +
+ S3AEncryptionMethods.SSE_KMS.getMethod());
+ }
+ return conf;
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_KMS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
new file mode 100644
index 0000000..8ce3a13
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Run the encryption tests against the Fast output stream.
+ * This verifies that both file writing paths can encrypt their data. This
+ * requires the SERVER_SIDE_ENCRYPTION_KEY to be set in auth-keys.xml for it
+ * to run.
+ */
+public class ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream
+ extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ if(StringUtils.isBlank(conf.get(Constants.SERVER_SIDE_ENCRYPTION_KEY))){
+ skip(Constants.SERVER_SIDE_ENCRYPTION_KEY+ " is not set for " +
+ S3AEncryptionMethods.SSE_KMS.getMethod());
+ }
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ conf.set(Constants.FAST_UPLOAD_BUFFER,
+ Constants.FAST_UPLOAD_BYTEBUFFER);
+ return conf;
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_KMS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3.java
new file mode 100644
index 0000000..33a252a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Concrete class that extends {@link AbstractTestS3AEncryption}
+ * and tests SSE-S3 encryption.
+ */
+public class ITestS3AEncryptionSSES3 extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ //must specify encryption key as empty because SSE-S3 does not allow it,
+ //nor can it be null.
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, "");
+ return conf;
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_S3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3BlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3BlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3BlockOutputStream.java
new file mode 100644
index 0000000..407601f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSES3BlockOutputStream.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Run the encryption tests against the block output stream.
+ */
+public class ITestS3AEncryptionSSES3BlockOutputStream
+ extends AbstractTestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ conf.set(Constants.FAST_UPLOAD_BUFFER,
+ Constants.FAST_UPLOAD_BYTEBUFFER);
+ //must specify encryption key as empty because SSE-S3 does not allow it,
+ //nor can it be null.
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, "");
+ return conf;
+ }
+
+ @Override
+ protected S3AEncryptionMethods getSSEAlgorithm() {
+ return S3AEncryptionMethods.SSE_S3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/839b690e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
index f9e9c6b..a5dc01a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
@@ -26,6 +26,7 @@ import java.io.FileNotFoundException;
import java.util.Collections;
import java.util.Date;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -34,6 +35,9 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
import org.junit.Test;
/**
@@ -48,7 +52,8 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
ObjectMetadata meta = new ObjectMetadata();
meta.setContentLength(1L);
meta.setLastModified(new Date(2L));
- when(s3.getObjectMetadata(BUCKET, key)).thenReturn(meta);
+ when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key))))
+ .thenReturn(meta);
FileStatus stat = fs.getFileStatus(path);
assertNotNull(stat);
assertEquals(fs.makeQualified(path), stat.getPath());
@@ -61,10 +66,13 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
public void testFakeDirectory() throws Exception {
Path path = new Path("/dir");
String key = path.toUri().getPath().substring(1);
- when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key))))
+ .thenThrow(NOT_FOUND);
ObjectMetadata meta = new ObjectMetadata();
meta.setContentLength(0L);
- when(s3.getObjectMetadata(BUCKET, key + "/")).thenReturn(meta);
+ when(s3.getObjectMetadata(argThat(
+ correctGetMetadataRequest(BUCKET, key + "/"))
+ )).thenReturn(meta);
FileStatus stat = fs.getFileStatus(path);
assertNotNull(stat);
assertEquals(fs.makeQualified(path), stat.getPath());
@@ -75,8 +83,11 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
public void testImplicitDirectory() throws Exception {
Path path = new Path("/dir");
String key = path.toUri().getPath().substring(1);
- when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
- when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key))))
+ .thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(
+ correctGetMetadataRequest(BUCKET, key + "/"))
+ )).thenThrow(NOT_FOUND);
ObjectListing objects = mock(ObjectListing.class);
when(objects.getCommonPrefixes()).thenReturn(
Collections.singletonList("dir/"));
@@ -93,8 +104,11 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
public void testRoot() throws Exception {
Path path = new Path("/");
String key = path.toUri().getPath().substring(1);
- when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
- when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key))))
+ .thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(
+ correctGetMetadataRequest(BUCKET, key + "/")
+ ))).thenThrow(NOT_FOUND);
ObjectListing objects = mock(ObjectListing.class);
when(objects.getCommonPrefixes()).thenReturn(
Collections.<String>emptyList());
@@ -112,8 +126,11 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
public void testNotFound() throws Exception {
Path path = new Path("/dir");
String key = path.toUri().getPath().substring(1);
- when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
- when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key))))
+ .thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(argThat(
+ correctGetMetadataRequest(BUCKET, key + "/")
+ ))).thenThrow(NOT_FOUND);
ObjectListing objects = mock(ObjectListing.class);
when(objects.getCommonPrefixes()).thenReturn(
Collections.<String>emptyList());
@@ -123,4 +140,26 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest {
exception.expect(FileNotFoundException.class);
fs.getFileStatus(path);
}
+
+ private Matcher<GetObjectMetadataRequest> correctGetMetadataRequest(
+ String bucket, String key) {
+ return new BaseMatcher<GetObjectMetadataRequest>() {
+
+ @Override
+ public void describeTo(Description description) {
+ description.appendText("bucket and key match");
+ }
+
+ @Override
+ public boolean matches(Object o) {
+ if(o instanceof GetObjectMetadataRequest) {
+ GetObjectMetadataRequest getObjectMetadataRequest =
+ (GetObjectMetadataRequest)o;
+ return getObjectMetadataRequest.getBucketName().equals(bucket)
+ && getObjectMetadataRequest.getKey().equals(key);
+ }
+ return false;
+ }
+ };
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-5325. Stateless ARMRMProxy
policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5325. Stateless ARMRMProxy policies implementation. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed1764cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed1764cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed1764cb
Branch: refs/heads/YARN-2915
Commit: ed1764cb98235db2fdd35a482f4cde3266e88f31
Parents: 6cf5730
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Oct 13 17:59:13 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../AbstractConfigurableFederationPolicy.java | 155 +++++
.../policies/ConfigurableFederationPolicy.java | 9 +-
.../FederationPolicyInitializationContext.java | 37 +-
...ionPolicyInitializationContextValidator.java | 28 +-
.../policies/FederationPolicyManager.java | 59 +-
.../amrmproxy/AbstractAMRMProxyPolicy.java | 47 ++
.../amrmproxy/BroadcastAMRMProxyPolicy.java | 85 +++
.../amrmproxy/FederationAMRMProxyPolicy.java | 25 +-
.../LocalityMulticastAMRMProxyPolicy.java | 583 +++++++++++++++++++
.../policies/amrmproxy/package-info.java | 1 -
.../policies/dao/WeightedPolicyInfo.java | 180 +++---
.../federation/policies/dao/package-info.java | 1 -
.../policies/exceptions/package-info.java | 1 -
.../federation/policies/package-info.java | 1 -
.../policies/router/AbstractRouterPolicy.java | 47 ++
.../router/BaseWeightedRouterPolicy.java | 150 -----
.../policies/router/FederationRouterPolicy.java | 5 +-
.../policies/router/LoadBasedRouterPolicy.java | 36 +-
.../policies/router/PriorityRouterPolicy.java | 19 +-
.../router/UniformRandomRouterPolicy.java | 28 +-
.../router/WeightedRandomRouterPolicy.java | 32 +-
.../policies/router/package-info.java | 1 -
.../resolver/AbstractSubClusterResolver.java | 4 +-
.../policies/BaseFederationPoliciesTest.java | 28 +-
...ionPolicyInitializationContextValidator.java | 25 +-
.../TestBroadcastAMRMProxyFederationPolicy.java | 112 ++++
.../TestLocalityMulticastAMRMProxyPolicy.java | 566 ++++++++++++++++++
.../router/TestLoadBasedRouterPolicy.java | 18 +-
.../router/TestPriorityRouterPolicy.java | 15 +-
.../router/TestWeightedRandomRouterPolicy.java | 35 +-
.../utils/FederationPoliciesTestUtil.java | 64 ++
.../src/test/resources/nodes | 6 +-
32 files changed, 1950 insertions(+), 453 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java
new file mode 100644
index 0000000..4cb9bbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * Base abstract class for a weighted {@link ConfigurableFederationPolicy}.
+ */
+public abstract class AbstractConfigurableFederationPolicy
+ implements ConfigurableFederationPolicy {
+
+ private WeightedPolicyInfo policyInfo = null;
+ private FederationPolicyInitializationContext policyContext;
+ private boolean isDirty;
+
+ public AbstractConfigurableFederationPolicy() {
+ }
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext initializationContext)
+ throws FederationPolicyInitializationException {
+ isDirty = true;
+ FederationPolicyInitializationContextValidator
+ .validate(initializationContext, this.getClass().getCanonicalName());
+
+ // perform consistency checks
+ WeightedPolicyInfo newPolicyInfo = WeightedPolicyInfo.fromByteBuffer(
+ initializationContext.getSubClusterPolicyConfiguration().getParams());
+
+ // if nothing has changed skip the rest of initialization
+ // and signal to childs that the reinit is free via isDirty var.
+ if (policyInfo != null && policyInfo.equals(newPolicyInfo)) {
+ isDirty = false;
+ return;
+ }
+
+ validate(newPolicyInfo);
+ setPolicyInfo(newPolicyInfo);
+ this.policyContext = initializationContext;
+ }
+
+ /**
+ * Overridable validation step for the policy configuration.
+ *
+ * @param newPolicyInfo the configuration to test.
+ *
+ * @throws FederationPolicyInitializationException if the configuration is not
+ * valid.
+ */
+ public void validate(WeightedPolicyInfo newPolicyInfo)
+ throws FederationPolicyInitializationException {
+ if (newPolicyInfo == null) {
+ throw new FederationPolicyInitializationException(
+ "The policy to " + "validate should not be null.");
+ }
+ }
+
+ /**
+ * Returns true whether the last reinitialization requires actual changes, or
+ * was "free" as the weights have not changed. This is used by subclasses
+ * overriding reinitialize and calling super.reinitialize() to know wheter to
+ * quit early.
+ *
+ * @return whether more work is needed to initialize.
+ */
+ public boolean getIsDirty() {
+ return isDirty;
+ }
+
+ /**
+ * Getter method for the configuration weights.
+ *
+ * @return the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public WeightedPolicyInfo getPolicyInfo() {
+ return policyInfo;
+ }
+
+ /**
+ * Setter method for the configuration weights.
+ *
+ * @param policyInfo the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public void setPolicyInfo(WeightedPolicyInfo policyInfo) {
+ this.policyInfo = policyInfo;
+ }
+
+ /**
+ * Getter method for the {@link FederationPolicyInitializationContext}.
+ *
+ * @return the context for this policy.
+ */
+ public FederationPolicyInitializationContext getPolicyContext() {
+ return policyContext;
+ }
+
+ /**
+ * Setter method for the {@link FederationPolicyInitializationContext}.
+ *
+ * @param policyContext the context to assign to this policy.
+ */
+ public void setPolicyContext(
+ FederationPolicyInitializationContext policyContext) {
+ this.policyContext = policyContext;
+ }
+
+ /**
+ * This methods gets active subclusters map from the {@code
+ * FederationStateStoreFacade} and validate it not being null/empty.
+ *
+ * @return the map of ids to info for all active subclusters.
+ *
+ * @throws YarnException if we can't get the list.
+ */
+ protected Map<SubClusterId, SubClusterInfo> getActiveSubclusters()
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getPolicyContext().getFederationStateStoreFacade().getSubClusters(true);
+
+ if (activeSubclusters == null || activeSubclusters.size() < 1) {
+ throw new NoActiveSubclustersException(
+ "Zero active subclusters, cannot pick where to send job.");
+ }
+ return activeSubclusters;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
index fd6ceea..5245772 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
@@ -31,14 +31,11 @@ public interface ConfigurableFederationPolicy {
* policies. The implementor should provide try-n-swap semantics, and retain
* state if possible.
*
- * @param federationPolicyInitializationContext the new context to provide to
- * implementor.
+ * @param policyContext the new context to provide to implementor.
*
* @throws FederationPolicyInitializationException in case the initialization
- * fails.
+ * fails.
*/
- void reinitialize(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext)
+ void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
index 9347fd0..46dd6eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.federation.policies;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
@@ -30,6 +31,7 @@ public class FederationPolicyInitializationContext {
private SubClusterPolicyConfiguration federationPolicyConfiguration;
private SubClusterResolver federationSubclusterResolver;
private FederationStateStoreFacade federationStateStoreFacade;
+ private SubClusterId homeSubcluster;
public FederationPolicyInitializationContext() {
federationPolicyConfiguration = null;
@@ -37,20 +39,19 @@ public class FederationPolicyInitializationContext {
federationStateStoreFacade = null;
}
- public FederationPolicyInitializationContext(SubClusterPolicyConfiguration
- policy, SubClusterResolver resolver, FederationStateStoreFacade
- storeFacade) {
+ public FederationPolicyInitializationContext(
+ SubClusterPolicyConfiguration policy, SubClusterResolver resolver,
+ FederationStateStoreFacade storeFacade) {
this.federationPolicyConfiguration = policy;
this.federationSubclusterResolver = resolver;
this.federationStateStoreFacade = storeFacade;
}
-
/**
* Getter for the {@link SubClusterPolicyConfiguration}.
*
* @return the {@link SubClusterPolicyConfiguration} to be used for
- * initialization.
+ * initialization.
*/
public SubClusterPolicyConfiguration getSubClusterPolicyConfiguration() {
return federationPolicyConfiguration;
@@ -59,8 +60,8 @@ public class FederationPolicyInitializationContext {
/**
* Setter for the {@link SubClusterPolicyConfiguration}.
*
- * @param fedPolicyConfiguration the {@link SubClusterPolicyConfiguration}
- * to be used for initialization.
+ * @param fedPolicyConfiguration the {@link SubClusterPolicyConfiguration} to
+ * be used for initialization.
*/
public void setSubClusterPolicyConfiguration(
SubClusterPolicyConfiguration fedPolicyConfiguration) {
@@ -80,7 +81,7 @@ public class FederationPolicyInitializationContext {
* Setter for the {@link SubClusterResolver}.
*
* @param federationSubclusterResolver the {@link SubClusterResolver} to be
- * used for initialization.
+ * used for initialization.
*/
public void setFederationSubclusterResolver(
SubClusterResolver federationSubclusterResolver) {
@@ -105,4 +106,24 @@ public class FederationPolicyInitializationContext {
FederationStateStoreFacade federationStateStoreFacade) {
this.federationStateStoreFacade = federationStateStoreFacade;
}
+
+ /**
+ * Returns the current home sub-cluster. Useful for default policy behaviors.
+ *
+ * @return the home sub-cluster.
+ */
+ public SubClusterId getHomeSubcluster() {
+ return homeSubcluster;
+ }
+
+ /**
+ * Sets in the context the home sub-cluster. Useful for default policy
+ * behaviors.
+ *
+ * @param homeSubcluster value to set.
+ */
+ public void setHomeSubcluster(SubClusterId homeSubcluster) {
+ this.homeSubcluster = homeSubcluster;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
index 31f83d4..1b83bbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
@@ -25,50 +25,44 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPo
public final class FederationPolicyInitializationContextValidator {
private FederationPolicyInitializationContextValidator() {
- //disable constructor per checkstyle
+ // disable constructor per checkstyle
}
public static void validate(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
- String myType) throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContext policyContext, String myType)
+ throws FederationPolicyInitializationException {
if (myType == null) {
- throw new FederationPolicyInitializationException("The myType parameter"
- + " should not be null.");
+ throw new FederationPolicyInitializationException(
+ "The myType parameter" + " should not be null.");
}
- if (federationPolicyInitializationContext == null) {
+ if (policyContext == null) {
throw new FederationPolicyInitializationException(
"The FederationPolicyInitializationContext provided is null. Cannot"
- + " reinitalize "
- + "successfully.");
+ + " reinitalize " + "successfully.");
}
- if (federationPolicyInitializationContext.getFederationStateStoreFacade()
- == null) {
+ if (policyContext.getFederationStateStoreFacade() == null) {
throw new FederationPolicyInitializationException(
"The FederationStateStoreFacade provided is null. Cannot"
+ " reinitalize successfully.");
}
- if (federationPolicyInitializationContext.getFederationSubclusterResolver()
- == null) {
+ if (policyContext.getFederationSubclusterResolver() == null) {
throw new FederationPolicyInitializationException(
"The FederationStateStoreFacase provided is null. Cannot"
+ " reinitalize successfully.");
}
- if (federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
- == null) {
+ if (policyContext.getSubClusterPolicyConfiguration() == null) {
throw new FederationPolicyInitializationException(
"The FederationSubclusterResolver provided is null. Cannot "
+ "reinitalize successfully.");
}
String intendedType =
- federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
- .getType();
+ policyContext.getSubClusterPolicyConfiguration().getType();
if (!myType.equals(intendedType)) {
throw new FederationPolicyInitializationException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
index e5dba63..39fdba3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
@@ -25,19 +25,19 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyCo
/**
*
* Implementors need to provide the ability to serliaze a policy and its
- * configuration as a {@link SubClusterPolicyConfiguration}, as well as
- * provide (re)initialization mechanics for the underlying
+ * configuration as a {@link SubClusterPolicyConfiguration}, as well as provide
+ * (re)initialization mechanics for the underlying
* {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
*
- * The serialization aspects are used by admin APIs or a policy engine to
- * store a serialized configuration in the {@code FederationStateStore},
- * while the getters methods are used to obtain a propertly inizialized
- * policy in the {@code Router} and {@code AMRMProxy} respectively.
+ * The serialization aspects are used by admin APIs or a policy engine to store
+ * a serialized configuration in the {@code FederationStateStore}, while the
+ * getters methods are used to obtain a propertly inizialized policy in the
+ * {@code Router} and {@code AMRMProxy} respectively.
*
- * This interface by design binds together
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} and
- * provide lifecycle support for serialization and deserialization, to reduce
- * configuration mistakes (combining incompatible policies).
+ * This interface by design binds together {@link FederationAMRMProxyPolicy} and
+ * {@link FederationRouterPolicy} and provide lifecycle support for
+ * serialization and deserialization, to reduce configuration mistakes
+ * (combining incompatible policies).
*
*/
public interface FederationPolicyManager {
@@ -50,23 +50,17 @@ public interface FederationPolicyManager {
* the implementors should attempt to reinitalize (retaining state). To affect
* a complete policy reset oldInstance should be null.
*
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
*
- * @return an updated {@link FederationAMRMProxyPolicy
- }.
+ * @return an updated {@link FederationAMRMProxyPolicy }.
*
* @throws FederationPolicyInitializationException if the initialization
- * cannot be completed
- * properly. The oldInstance
- * should be still valid in
- * case of failed
- * initialization.
+ * cannot be completed properly. The oldInstance should be still
+ * valid in case of failed initialization.
*/
FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationAMRMProxyPolicy oldInstance)
throws FederationPolicyInitializationException;
@@ -78,21 +72,17 @@ public interface FederationPolicyManager {
* implementors should attempt to reinitalize (retaining state). To affect a
* complete policy reset oldInstance shoulb be set to null.
*
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
*
* @return an updated {@link FederationRouterPolicy}.
*
* @throws FederationPolicyInitializationException if the initalization cannot
- * be completed properly. The
- * oldInstance should be still
- * valid in case of failed
- * initialization.
+ * be completed properly. The oldInstance should be still valid in
+ * case of failed initialization.
*/
FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationRouterPolicy oldInstance)
throws FederationPolicyInitializationException;
@@ -102,23 +92,24 @@ public interface FederationPolicyManager {
* store.
*
* @return a valid policy configuration representing this object
- * parametrization.
+ * parametrization.
*
* @throws FederationPolicyInitializationException if the current state cannot
- * be serialized properly
+ * be serialized properly
*/
SubClusterPolicyConfiguration serializeConf()
throws FederationPolicyInitializationException;
-
/**
* This method returns the queue this policy is configured for.
+ *
* @return the name of the queue.
*/
String getQueue();
/**
* This methods provides a setter for the queue this policy is specified for.
+ *
* @param queue the name of the queue.
*/
void setQueue(String queue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java
new file mode 100644
index 0000000..e853744
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+
+/**
+ * Base abstract class for {@link FederationAMRMProxyPolicy} implementations,
+ * that provides common validation for reinitialization.
+ */
+public abstract class AbstractAMRMProxyPolicy extends
+ AbstractConfigurableFederationPolicy implements FederationAMRMProxyPolicy {
+
+ @Override
+ public void validate(WeightedPolicyInfo newPolicyInfo)
+ throws FederationPolicyInitializationException {
+ super.validate(newPolicyInfo);
+ Map<SubClusterIdInfo, Float> newWeights =
+ newPolicyInfo.getAMRMPolicyWeights();
+ if (newWeights == null || newWeights.size() < 1) {
+ throw new FederationPolicyInitializationException(
+ "Weight vector cannot be null/empty.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
new file mode 100644
index 0000000..679f4d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.UnknownSubclusterException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} that simply
+ * broadcasts each {@link ResourceRequest} to all the available sub-clusters.
+ */
+public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ private Set<SubClusterId> knownClusterIds = new HashSet<>();
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+ // overrides initialize to avoid weight checks that do no apply for
+ // this policy.
+ FederationPolicyInitializationContextValidator
+ .validate(policyContext, this.getClass().getCanonicalName());
+ setPolicyContext(policyContext);
+ }
+
+ @Override
+ public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests) throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ Map<SubClusterId, List<ResourceRequest>> answer = new HashMap<>();
+
+ // simply broadcast the resource request to all sub-clusters
+ for (SubClusterId subClusterId : activeSubclusters.keySet()) {
+ answer.put(subClusterId, resourceRequests);
+ knownClusterIds.add(subClusterId);
+ }
+
+ return answer;
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+ if (!knownClusterIds.contains(subClusterId)) {
+ throw new UnknownSubclusterException(
+ "The response is received from a subcluster that is unknown to this "
+ + "policy.");
+ }
+ // stateless policy does not care about responses
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
index 4a3305c..0541df4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
@@ -17,18 +17,18 @@
package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import java.util.List;
-import java.util.Map;
-
/**
- * Implementors of this interface provide logic to split the list of {@link
- * ResourceRequest}s received by the AM among various RMs.
+ * Implementors of this interface provide logic to split the list of
+ * {@link ResourceRequest}s received by the AM among various RMs.
*/
public interface FederationAMRMProxyPolicy
extends ConfigurableFederationPolicy {
@@ -37,18 +37,17 @@ public interface FederationAMRMProxyPolicy
* Splits the {@link ResourceRequest}s from the client across one or more
* sub-clusters based on the policy semantics (e.g., broadcast, load-based).
*
- * @param resourceRequests the list of {@link ResourceRequest}s from the
- * AM to be split
+ * @param resourceRequests the list of {@link ResourceRequest}s from the AM to
+ * be split
*
* @return map of sub-cluster as identified by {@link SubClusterId} to the
- * list of {@link ResourceRequest}s that should be forwarded to it
+ * list of {@link ResourceRequest}s that should be forwarded to it
*
* @throws YarnException in case the request is malformed or no viable
- * sub-clusters can be found.
+ * sub-clusters can be found.
*/
Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
- List<ResourceRequest> resourceRequests)
- throws YarnException;
+ List<ResourceRequest> resourceRequests) throws YarnException;
/**
* This method should be invoked to notify the policy about responses being
@@ -60,7 +59,7 @@ public interface FederationAMRMProxyPolicy
*
* @throws YarnException in case the response is not valid
*/
- void notifyOfResponse(SubClusterId subClusterId,
- AllocateResponse response) throws YarnException;
+ void notifyOfResponse(SubClusterId subClusterId, AllocateResponse response)
+ throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
new file mode 100644
index 0000000..283f89e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -0,0 +1,583 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} interface that
+ * carefully multicasts the requests with the following behavior:
+ *
+ * <p>
+ * Host localized {@link ResourceRequest}s are always forwarded to the RM that
+ * owns the corresponding node, based on the feedback of a
+ * {@link SubClusterResolver}. If the {@link SubClusterResolver} cannot resolve
+ * this node we default to forwarding the {@link ResourceRequest} to the home
+ * sub-cluster.
+ * </p>
+ *
+ * <p>
+ * Rack localized {@link ResourceRequest}s are forwarded to the RMs that owns
+ * the corresponding rack. Note that in some deployments each rack could be
+ * striped across multiple RMs. Thsi policy respects that. If the
+ * {@link SubClusterResolver} cannot resolve this rack we default to forwarding
+ * the {@link ResourceRequest} to the home sub-cluster.
+ * </p>
+ *
+ * <p>
+ * ANY requests corresponding to node/rack local requests are forwarded only to
+ * the set of RMs that owns the corresponding localized requests. The number of
+ * containers listed in each ANY is proportional to the number of localized
+ * container requests (associated to this ANY via the same allocateRequestId).
+ * </p>
+ *
+ * <p>
+ * ANY that are not associated to node/rack local requests are split among RMs
+ * based on the "weights" in the {@link WeightedPolicyInfo} configuration *and*
+ * headroom information. The {@code headroomAlpha} parameter of the policy
+ * configuration indicates how much headroom contributes to the splitting
+ * choice. Value of 1.0f indicates the weights are interpreted only as 0/1
+ * boolean but all splitting is based on the advertised headroom (fallback to
+ * 1/N for RMs that we don't have headroom info from). An {@code headroomAlpha}
+ * value of 0.0f means headroom is ignored and all splitting decisions are
+ * proportional to the "weights" in the configuration of the policy.
+ * </p>
+ *
+ * <p>
+ * ANY of zero size are forwarded to all known subclusters (i.e., subclusters
+ * where we scheduled containers before), as they may represent a user attempt
+ * to cancel a previous request (and we are mostly stateless now, so should
+ * forward to all known RMs).
+ * </p>
+ *
+ * <p>
+ * Invariants:
+ * </p>
+ *
+ * <p>
+ * The policy always excludes non-active RMs.
+ * </p>
+ *
+ * <p>
+ * The policy always excludes RMs that do not appear in the policy configuration
+ * weights, or have a weight of 0 (even if localized resources explicit refer to
+ * it).
+ * </p>
+ *
+ * <p>
+ * (Bar rounding to closest ceiling of fractional containers) The sum of
+ * requests made to multiple RMs at the ANY level "adds-up" to the user request.
+ * The maximum possible excess in a given request is a number of containers less
+ * or equal to number of sub-clusters in the federation.
+ * </p>
+ */
+public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(LocalityMulticastAMRMProxyPolicy.class);
+
+ private Map<SubClusterId, Float> weights;
+ private SubClusterResolver resolver;
+
+ private Map<SubClusterId, Resource> headroom;
+ private float hrAlpha;
+ private FederationStateStoreFacade federationFacade;
+ private AllocationBookkeeper bookkeeper;
+ private SubClusterId homeSubcluster;
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+
+ // save reference to old weights
+ WeightedPolicyInfo tempPolicy = getPolicyInfo();
+
+ super.reinitialize(policyContext);
+ if (!getIsDirty()) {
+ return;
+ }
+
+ Map<SubClusterId, Float> newWeightsConverted = new HashMap<>();
+ boolean allInactive = true;
+ WeightedPolicyInfo policy = getPolicyInfo();
+ if (policy.getAMRMPolicyWeights() == null
+ || policy.getAMRMPolicyWeights().size() == 0) {
+ allInactive = false;
+ } else {
+ for (Map.Entry<SubClusterIdInfo, Float> e : policy.getAMRMPolicyWeights()
+ .entrySet()) {
+ if (e.getValue() > 0) {
+ allInactive = false;
+ }
+ newWeightsConverted.put(e.getKey().toId(), e.getValue());
+ }
+ }
+ if (allInactive) {
+ // reset the policyInfo and throw
+ setPolicyInfo(tempPolicy);
+ throw new FederationPolicyInitializationException(
+ "The weights used to configure "
+ + "this policy are all set to zero! (no ResourceRequest could be "
+ + "forwarded with this setting.)");
+ }
+
+ if (policyContext.getHomeSubcluster() == null) {
+ setPolicyInfo(tempPolicy);
+ throw new FederationPolicyInitializationException("The homeSubcluster "
+ + "filed in the context must be initialized to use this policy");
+ }
+
+ weights = newWeightsConverted;
+ resolver = policyContext.getFederationSubclusterResolver();
+
+ if (headroom == null) {
+ headroom = new ConcurrentHashMap<>();
+ }
+ hrAlpha = policy.getHeadroomAlpha();
+
+ this.federationFacade =
+ policyContext.getFederationStateStoreFacade();
+ this.bookkeeper = new AllocationBookkeeper();
+ this.homeSubcluster = policyContext.getHomeSubcluster();
+
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+ // stateless policy does not care about responses except tracking headroom
+ headroom.put(subClusterId, response.getAvailableResources());
+ }
+
+ @Override
+ public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests) throws YarnException {
+
+ // object used to accumulate statistics about the answer, initialize with
+ // active subclusters.
+ bookkeeper.reinitialize(federationFacade.getSubClusters(true));
+
+ List<ResourceRequest> nonLocalizedRequests =
+ new ArrayList<ResourceRequest>();
+
+ SubClusterId targetId = null;
+ Set<SubClusterId> targetIds = null;
+
+ // if the RR is resolved to a local subcluster add it directly (node and
+ // resolvable racks)
+ for (ResourceRequest rr : resourceRequests) {
+ targetId = null;
+ targetIds = null;
+
+ // Handle: ANY (accumulated for later)
+ if (ResourceRequest.isAnyLocation(rr.getResourceName())) {
+ nonLocalizedRequests.add(rr);
+ continue;
+ }
+
+ // Handle "node" requests
+ try {
+ targetId = resolver.getSubClusterForNode(rr.getResourceName());
+ } catch (YarnException e) {
+ // this might happen as we can't differentiate node from rack names
+ // we log altogether later
+ }
+ if (bookkeeper.isActiveAndEnabled(targetId)) {
+ bookkeeper.addLocalizedNodeRR(targetId, rr);
+ continue;
+ }
+
+ // Handle "rack" requests
+ try {
+ targetIds = resolver.getSubClustersForRack(rr.getResourceName());
+ } catch (YarnException e) {
+ // this might happen as we can't differentiate node from rack names
+ // we log altogether later
+ }
+ if (targetIds != null && targetIds.size() > 0) {
+ for (SubClusterId tid : targetIds) {
+ if (bookkeeper.isActiveAndEnabled(tid)) {
+ bookkeeper.addRackRR(tid, rr);
+ }
+ }
+ continue;
+ }
+
+ // Handle node/rack requests that the SubClusterResolver cannot map to
+ // any cluster. Defaulting to home subcluster.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ERROR resolving sub-cluster for resourceName: "
+ + rr.getResourceName() + " we are falling back to homeSubCluster:"
+ + homeSubcluster);
+ }
+
+ // If home-subcluster is not active, ignore node/rack request
+ if (bookkeeper.isActiveAndEnabled(homeSubcluster)) {
+ bookkeeper.addLocalizedNodeRR(homeSubcluster, rr);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The homeSubCluster (" + homeSubcluster + ") we are "
+ + "defaulting to is not active, the ResourceRequest "
+ + "will be ignored.");
+ }
+ }
+ }
+
+ // handle all non-localized requests (ANY)
+ splitAnyRequests(nonLocalizedRequests, bookkeeper);
+
+ return bookkeeper.getAnswer();
+ }
+
+ /**
+ * It splits a list of non-localized resource requests among sub-clusters.
+ */
+ private void splitAnyRequests(List<ResourceRequest> originalResourceRequests,
+ AllocationBookkeeper allocationBookkeeper) throws YarnException {
+
+ for (ResourceRequest resourceRequest : originalResourceRequests) {
+
+ // FIRST: pick the target set of subclusters (based on whether this RR
+ // is associated with other localized requests via an allocationId)
+ Long allocationId = resourceRequest.getAllocationRequestId();
+ Set<SubClusterId> targetSubclusters;
+ if (allocationBookkeeper.getSubClustersForId(allocationId) != null) {
+ targetSubclusters =
+ allocationBookkeeper.getSubClustersForId(allocationId);
+ } else {
+ targetSubclusters = allocationBookkeeper.getActiveAndEnabledSC();
+ }
+
+ // SECOND: pick how much to ask to each RM for each request
+ splitIndividualAny(resourceRequest, targetSubclusters,
+ allocationBookkeeper);
+ }
+ }
+
+ /**
+ * Return a projection of this ANY {@link ResourceRequest} that belongs to
+ * this sub-cluster. This is done based on the "count" of the containers that
+ * require locality in each sublcuster (if any) or based on the "weights" and
+ * headroom.
+ */
+ private void splitIndividualAny(ResourceRequest originalResourceRequest,
+ Set<SubClusterId> targetSubclusters,
+ AllocationBookkeeper allocationBookkeeper) {
+
+ long allocationId = originalResourceRequest.getAllocationRequestId();
+
+ for (SubClusterId targetId : targetSubclusters) {
+ float numContainer = originalResourceRequest.getNumContainers();
+
+ // If the ANY request has 0 containers to begin with we must forward it to
+ // any RM we have previously contacted (this might be the user way
+ // to cancel a previous request).
+ if (numContainer == 0 && headroom.containsKey(targetId)) {
+ allocationBookkeeper.addAnyRR(targetId, originalResourceRequest);
+ }
+
+ // If ANY is associated with localized asks, split based on their ratio
+ if (allocationBookkeeper.getSubClustersForId(allocationId) != null) {
+ float localityBasedWeight = getLocalityBasedWeighting(allocationId,
+ targetId, allocationBookkeeper);
+ numContainer = numContainer * localityBasedWeight;
+ } else {
+ // split ANY based on load and policy configuration
+ float headroomWeighting =
+ getHeadroomWeighting(targetId, allocationBookkeeper);
+ float policyWeighting =
+ getPolicyConfigWeighting(targetId, allocationBookkeeper);
+ // hrAlpha controls how much headroom influencing decision
+ numContainer = numContainer
+ * (hrAlpha * headroomWeighting + (1 - hrAlpha) * policyWeighting);
+ }
+
+ // if the calculated request is non-empty add it to the answer
+ if (numContainer > 0) {
+ ResourceRequest out =
+ ResourceRequest.newInstance(originalResourceRequest.getPriority(),
+ originalResourceRequest.getResourceName(),
+ originalResourceRequest.getCapability(),
+ originalResourceRequest.getNumContainers(),
+ originalResourceRequest.getRelaxLocality(),
+ originalResourceRequest.getNodeLabelExpression(),
+ originalResourceRequest.getExecutionTypeRequest());
+ out.setAllocationRequestId(allocationId);
+ out.setNumContainers((int) Math.ceil(numContainer));
+ if (out.isAnyLocation(out.getResourceName())) {
+ allocationBookkeeper.addAnyRR(targetId, out);
+ } else {
+ allocationBookkeeper.addRackRR(targetId, out);
+ }
+ }
+ }
+ }
+
+ /**
+ * Compute the weight to assign to a subcluster based on how many local
+ * requests a subcluster is target of.
+ */
+ private float getLocalityBasedWeighting(long reqId, SubClusterId targetId,
+ AllocationBookkeeper allocationBookkeeper) {
+ float totWeight = allocationBookkeeper.getTotNumLocalizedContainers();
+ float localWeight =
+ allocationBookkeeper.getNumLocalizedContainers(reqId, targetId);
+ return totWeight > 0 ? localWeight / totWeight : 0;
+ }
+
+ /**
+ * Compute the "weighting" to give to a sublcuster based on the configured
+ * policy weights (for the active subclusters).
+ */
+ private float getPolicyConfigWeighting(SubClusterId targetId,
+ AllocationBookkeeper allocationBookkeeper) {
+ float totWeight = allocationBookkeeper.totPolicyWeight;
+ Float localWeight = weights.get(targetId);
+ return (localWeight != null && totWeight > 0) ? localWeight / totWeight : 0;
+ }
+
+ /**
+ * Compute the weighting based on available headroom. This is proportional to
+ * the available headroom memory announced by RM, or to 1/N for RMs we have
+ * not seen yet. If all RMs report zero headroom, we fallback to 1/N again.
+ */
+ private float getHeadroomWeighting(SubClusterId targetId,
+ AllocationBookkeeper allocationBookkeeper) {
+
+ // baseline weight for all RMs
+ float headroomWeighting =
+ 1 / (float) allocationBookkeeper.getActiveAndEnabledSC().size();
+
+ // if we have headroom infomration for this sub-cluster (and we are safe
+ // from /0 issues)
+ if (headroom.containsKey(targetId)
+ && allocationBookkeeper.totHeadroomMemory > 0) {
+ // compute which portion of the RMs that are active/enabled have reported
+ // their headroom (needed as adjustment factor)
+ // (note: getActiveAndEnabledSC should never be null/zero)
+ float ratioHeadroomKnown = allocationBookkeeper.totHeadRoomEnabledRMs
+ / (float) allocationBookkeeper.getActiveAndEnabledSC().size();
+
+ // headroomWeighting is the ratio of headroom memory in the targetId
+ // cluster / total memory. The ratioHeadroomKnown factor is applied to
+ // adjust for missing information and ensure sum of allocated containers
+ // closely approximate what the user asked (small excess).
+ headroomWeighting = (headroom.get(targetId).getMemorySize()
+ / allocationBookkeeper.totHeadroomMemory) * (ratioHeadroomKnown);
+ }
+ return headroomWeighting;
+ }
+
+ /**
+ * This helper class is used to book-keep the requests made to each
+ * subcluster, and maintain useful statistics to split ANY requests.
+ */
+ private final class AllocationBookkeeper {
+
+ // the answer being accumulated
+ private Map<SubClusterId, List<ResourceRequest>> answer = new TreeMap<>();
+
+ // stores how many containers we have allocated in each RM for localized
+ // asks, used to correctly "spread" the corresponding ANY
+ private Map<Long, Map<SubClusterId, AtomicLong>> countContainersPerRM =
+ new HashMap<>();
+
+ private Set<SubClusterId> activeAndEnabledSC = new HashSet<>();
+ private long totNumLocalizedContainers = 0;
+ private float totHeadroomMemory = 0;
+ private int totHeadRoomEnabledRMs = 0;
+ private float totPolicyWeight = 0;
+
+ private void reinitialize(
+ Map<SubClusterId, SubClusterInfo> activeSubclusters)
+ throws YarnException {
+
+ // reset data structures
+ answer.clear();
+ countContainersPerRM.clear();
+ activeAndEnabledSC.clear();
+ totNumLocalizedContainers = 0;
+ totHeadroomMemory = 0;
+ totHeadRoomEnabledRMs = 0;
+ totPolicyWeight = 0;
+
+ // pre-compute the set of subclusters that are both active and enabled by
+ // the policy weights, and accumulate their total weight
+ for (Map.Entry<SubClusterId, Float> entry : weights.entrySet()) {
+ if (entry.getValue() > 0
+ && activeSubclusters.containsKey(entry.getKey())) {
+ activeAndEnabledSC.add(entry.getKey());
+ totPolicyWeight += entry.getValue();
+ }
+ }
+
+ if (activeAndEnabledSC.size() < 1) {
+ throw new NoActiveSubclustersException(
+ "None of the subclusters enabled in this policy (weight>0) are "
+ + "currently active we cannot forward the ResourceRequest(s)");
+ }
+
+ // pre-compute headroom-based weights for active/enabled subclusters
+ for (Map.Entry<SubClusterId, Resource> r : headroom.entrySet()) {
+ if (activeAndEnabledSC.contains(r.getKey())) {
+ totHeadroomMemory += r.getValue().getMemorySize();
+ totHeadRoomEnabledRMs++;
+ }
+ }
+
+ }
+
+ /**
+ * Add to the answer a localized node request, and keeps track of statistics
+ * on a per-allocation-id and per-subcluster bases.
+ */
+ private void addLocalizedNodeRR(SubClusterId targetId, ResourceRequest rr) {
+ Preconditions.checkArgument(!rr.isAnyLocation(rr.getResourceName()));
+
+ if (!countContainersPerRM.containsKey(rr.getAllocationRequestId())) {
+ countContainersPerRM.put(rr.getAllocationRequestId(), new HashMap<>());
+ }
+ if (!countContainersPerRM.get(rr.getAllocationRequestId())
+ .containsKey(targetId)) {
+ countContainersPerRM.get(rr.getAllocationRequestId()).put(targetId,
+ new AtomicLong(0));
+ }
+ countContainersPerRM.get(rr.getAllocationRequestId()).get(targetId)
+ .addAndGet(rr.getNumContainers());
+
+ totNumLocalizedContainers += rr.getNumContainers();
+
+ internalAddToAnswer(targetId, rr);
+ }
+
+ /**
+ * Add a rack-local request to the final asnwer.
+ */
+ public void addRackRR(SubClusterId targetId, ResourceRequest rr) {
+ Preconditions.checkArgument(!rr.isAnyLocation(rr.getResourceName()));
+ internalAddToAnswer(targetId, rr);
+ }
+
+ /**
+ * Add an ANY request to the final answer.
+ */
+ private void addAnyRR(SubClusterId targetId, ResourceRequest rr) {
+ Preconditions.checkArgument(rr.isAnyLocation(rr.getResourceName()));
+ internalAddToAnswer(targetId, rr);
+ }
+
+ private void internalAddToAnswer(SubClusterId targetId,
+ ResourceRequest partialRR) {
+ if (!answer.containsKey(targetId)) {
+ answer.put(targetId, new ArrayList<ResourceRequest>());
+ }
+ answer.get(targetId).add(partialRR);
+ }
+
+ /**
+ * Return all known subclusters associated with an allocation id.
+ *
+ * @param allocationId the allocation id considered
+ *
+ * @return the list of {@link SubClusterId}s associated with this allocation
+ * id
+ */
+ private Set<SubClusterId> getSubClustersForId(long allocationId) {
+ if (countContainersPerRM.get(allocationId) == null) {
+ return null;
+ }
+ return countContainersPerRM.get(allocationId).keySet();
+ }
+
+ /**
+ * Return the answer accumulated so far.
+ *
+ * @return the answer
+ */
+ private Map<SubClusterId, List<ResourceRequest>> getAnswer() {
+ return answer;
+ }
+
+ /**
+ * Return the set of sub-clusters that are both active and allowed by our
+ * policy (weight > 0).
+ *
+ * @return a set of active and enabled {@link SubClusterId}s
+ */
+ private Set<SubClusterId> getActiveAndEnabledSC() {
+ return activeAndEnabledSC;
+ }
+
+ /**
+ * Return the total number of container coming from localized requests.
+ */
+ private long getTotNumLocalizedContainers() {
+ return totNumLocalizedContainers;
+ }
+
+ /**
+ * Returns the number of containers matching an allocation Id that are
+ * localized in the targetId subcluster.
+ */
+ private long getNumLocalizedContainers(long allocationId,
+ SubClusterId targetId) {
+ AtomicLong c = countContainersPerRM.get(allocationId).get(targetId);
+ return c == null ? 0 : c.get();
+ }
+
+ /**
+ * Returns true is the subcluster request is both active and enabled.
+ */
+ private boolean isActiveAndEnabled(SubClusterId targetId) {
+ if (targetId == null) {
+ return false;
+ } else {
+ return getActiveAndEnabledSC().contains(targetId);
+ }
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
index 99da20b..ef72647 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
@@ -17,4 +17,3 @@
*/
/** AMRMPRoxy policies. **/
package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
index a0fa37f..62eb03b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
@@ -17,10 +17,19 @@
package org.apache.hadoop.yarn.server.federation.policies.dao;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.api.json.JSONJAXBContext;
-import com.sun.jersey.api.json.JSONMarshaller;
-import com.sun.jersey.api.json.JSONUnmarshaller;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -29,24 +38,16 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.api.json.JSONUnmarshaller;
/**
* This is a DAO class for the configuration of parameteres for federation
* policies. This generalizes several possible configurations as two lists of
- * {@link SubClusterIdInfo} and corresponding weights as a
- * {@link Float}. The interpretation of the weight is left to the logic in
- * the policy.
+ * {@link SubClusterIdInfo} and corresponding weights as a {@link Float}. The
+ * interpretation of the weight is left to the logic in the policy.
*/
@InterfaceAudience.Private
@@ -57,12 +58,14 @@ public class WeightedPolicyInfo {
private static final Logger LOG =
LoggerFactory.getLogger(WeightedPolicyInfo.class);
-
+ private static JSONJAXBContext jsonjaxbContext = initContext();
private Map<SubClusterIdInfo, Float> routerPolicyWeights = new HashMap<>();
private Map<SubClusterIdInfo, Float> amrmPolicyWeights = new HashMap<>();
private float headroomAlpha;
- private static JSONJAXBContext jsonjaxbContext = initContext();
+ public WeightedPolicyInfo() {
+ // JAXB needs this
+ }
private static JSONJAXBContext initContext() {
try {
@@ -74,46 +77,6 @@ public class WeightedPolicyInfo {
return null;
}
- public WeightedPolicyInfo() {
- //JAXB needs this
- }
-
- /**
- * Setter method for Router weights.
- *
- * @param policyWeights the router weights.
- */
- public void setRouterPolicyWeights(
- Map<SubClusterIdInfo, Float> policyWeights) {
- this.routerPolicyWeights = policyWeights;
- }
-
- /**
- * Setter method for ARMRMProxy weights.
- *
- * @param policyWeights the amrmproxy weights.
- */
- public void setAMRMPolicyWeights(
- Map<SubClusterIdInfo, Float> policyWeights) {
- this.amrmPolicyWeights = policyWeights;
- }
-
- /**
- * Getter of the router weights.
- * @return the router weights.
- */
- public Map<SubClusterIdInfo, Float> getRouterPolicyWeights() {
- return routerPolicyWeights;
- }
-
- /**
- * Getter for AMRMProxy weights.
- * @return the AMRMProxy weights.
- */
- public Map<SubClusterIdInfo, Float> getAMRMPolicyWeights() {
- return amrmPolicyWeights;
- }
-
/**
* Deserializes a {@link WeightedPolicyInfo} from a byte UTF-8 JSON
* representation.
@@ -123,14 +86,14 @@ public class WeightedPolicyInfo {
* @return the {@link WeightedPolicyInfo} represented.
*
* @throws FederationPolicyInitializationException if a deserializaiton error
- * occurs.
+ * occurs.
*/
public static WeightedPolicyInfo fromByteBuffer(ByteBuffer bb)
throws FederationPolicyInitializationException {
if (jsonjaxbContext == null) {
- throw new FederationPolicyInitializationException("JSONJAXBContext should"
- + " not be null.");
+ throw new FederationPolicyInitializationException(
+ "JSONJAXBContext should" + " not be null.");
}
try {
@@ -139,9 +102,8 @@ public class WeightedPolicyInfo {
bb.get(bytes);
String params = new String(bytes, Charset.forName("UTF-8"));
- WeightedPolicyInfo weightedPolicyInfo = unmarshaller
- .unmarshalFromJSON(new StringReader(params),
- WeightedPolicyInfo.class);
+ WeightedPolicyInfo weightedPolicyInfo = unmarshaller.unmarshalFromJSON(
+ new StringReader(params), WeightedPolicyInfo.class);
return weightedPolicyInfo;
} catch (JAXBException j) {
throw new FederationPolicyInitializationException(j);
@@ -149,19 +111,56 @@ public class WeightedPolicyInfo {
}
/**
- * Converts the policy into a byte array representation in the input {@link
- * ByteBuffer}.
+ * Getter of the router weights.
+ *
+ * @return the router weights.
+ */
+ public Map<SubClusterIdInfo, Float> getRouterPolicyWeights() {
+ return routerPolicyWeights;
+ }
+
+ /**
+ * Setter method for Router weights.
+ *
+ * @param policyWeights the router weights.
+ */
+ public void setRouterPolicyWeights(
+ Map<SubClusterIdInfo, Float> policyWeights) {
+ this.routerPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Getter for AMRMProxy weights.
+ *
+ * @return the AMRMProxy weights.
+ */
+ public Map<SubClusterIdInfo, Float> getAMRMPolicyWeights() {
+ return amrmPolicyWeights;
+ }
+
+ /**
+ * Setter method for ARMRMProxy weights.
+ *
+ * @param policyWeights the amrmproxy weights.
+ */
+ public void setAMRMPolicyWeights(Map<SubClusterIdInfo, Float> policyWeights) {
+ this.amrmPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Converts the policy into a byte array representation in the input
+ * {@link ByteBuffer}.
*
* @return byte array representation of this policy configuration.
*
* @throws FederationPolicyInitializationException if a serialization error
- * occurs.
+ * occurs.
*/
public ByteBuffer toByteBuffer()
throws FederationPolicyInitializationException {
if (jsonjaxbContext == null) {
- throw new FederationPolicyInitializationException("JSONJAXBContext should"
- + " not be null.");
+ throw new FederationPolicyInitializationException(
+ "JSONJAXBContext should" + " not be null.");
}
try {
String s = toJSONString();
@@ -186,22 +185,21 @@ public class WeightedPolicyInfo {
return false;
}
- WeightedPolicyInfo otherPolicy =
- (WeightedPolicyInfo) other;
+ WeightedPolicyInfo otherPolicy = (WeightedPolicyInfo) other;
Map<SubClusterIdInfo, Float> otherAMRMWeights =
otherPolicy.getAMRMPolicyWeights();
Map<SubClusterIdInfo, Float> otherRouterWeights =
otherPolicy.getRouterPolicyWeights();
- boolean amrmWeightsMatch = otherAMRMWeights != null &&
- getAMRMPolicyWeights() != null &&
- CollectionUtils.isEqualCollection(otherAMRMWeights.entrySet(),
- getAMRMPolicyWeights().entrySet());
+ boolean amrmWeightsMatch =
+ otherAMRMWeights != null && getAMRMPolicyWeights() != null
+ && CollectionUtils.isEqualCollection(otherAMRMWeights.entrySet(),
+ getAMRMPolicyWeights().entrySet());
- boolean routerWeightsMatch = otherRouterWeights != null &&
- getRouterPolicyWeights() != null &&
- CollectionUtils.isEqualCollection(otherRouterWeights.entrySet(),
- getRouterPolicyWeights().entrySet());
+ boolean routerWeightsMatch =
+ otherRouterWeights != null && getRouterPolicyWeights() != null
+ && CollectionUtils.isEqualCollection(otherRouterWeights.entrySet(),
+ getRouterPolicyWeights().entrySet());
return amrmWeightsMatch && routerWeightsMatch;
}
@@ -215,10 +213,10 @@ public class WeightedPolicyInfo {
* Return the parameter headroomAlpha, used by policies that balance
* weight-based and load-based considerations in their decisions.
*
- * For policies that use this parameter, values close to 1 indicate that
- * most of the decision should be based on currently observed headroom from
- * various sub-clusters, values close to zero, indicate that the decision
- * should be mostly based on weights and practically ignore current load.
+ * For policies that use this parameter, values close to 1 indicate that most
+ * of the decision should be based on currently observed headroom from various
+ * sub-clusters, values close to zero, indicate that the decision should be
+ * mostly based on weights and practically ignore current load.
*
* @return the value of headroomAlpha.
*/
@@ -227,13 +225,13 @@ public class WeightedPolicyInfo {
}
/**
- * Set the parameter headroomAlpha, used by policies that balance
- * weight-based and load-based considerations in their decisions.
+ * Set the parameter headroomAlpha, used by policies that balance weight-based
+ * and load-based considerations in their decisions.
*
- * For policies that use this parameter, values close to 1 indicate that
- * most of the decision should be based on currently observed headroom from
- * various sub-clusters, values close to zero, indicate that the decision
- * should be mostly based on weights and practically ignore current load.
+ * For policies that use this parameter, values close to 1 indicate that most
+ * of the decision should be based on currently observed headroom from various
+ * sub-clusters, values close to zero, indicate that the decision should be
+ * mostly based on weights and practically ignore current load.
*
* @param headroomAlpha the value to use for balancing.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
index 43f5b83..c292e52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
@@ -17,4 +17,3 @@
*/
/** DAO objects for serializing/deserializing policy configurations. **/
package org.apache.hadoop.yarn.server.federation.policies.dao;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
index 3318da9..ad2d543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
@@ -17,4 +17,3 @@
*/
/** Exceptions for policies. **/
package org.apache.hadoop.yarn.server.federation.policies.exceptions;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
index 7d9a121..fa3fcc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
@@ -17,4 +17,3 @@
*/
/** Federation Policies. **/
package org.apache.hadoop.yarn.server.federation.policies;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
new file mode 100644
index 0000000..f49af1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+
+/**
+ * Base abstract class for {@link FederationRouterPolicy} implementations, that
+ * provides common validation for reinitialization.
+ */
+public abstract class AbstractRouterPolicy extends
+ AbstractConfigurableFederationPolicy implements FederationRouterPolicy {
+
+ @Override
+ public void validate(WeightedPolicyInfo newPolicyInfo)
+ throws FederationPolicyInitializationException {
+ super.validate(newPolicyInfo);
+ Map<SubClusterIdInfo, Float> newWeights =
+ newPolicyInfo.getRouterPolicyWeights();
+ if (newWeights == null || newWeights.size() < 1) {
+ throw new FederationPolicyInitializationException(
+ "Weight vector cannot be null/empty.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
deleted file mode 100644
index e888979..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies.router;
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-
-import java.util.Map;
-
-/**
- * Abstract class provides common validation of reinitialize(), for all
- * policies that are "weight-based".
- */
-public abstract class BaseWeightedRouterPolicy
- implements FederationRouterPolicy {
-
- private WeightedPolicyInfo policyInfo = null;
- private FederationPolicyInitializationContext policyContext;
-
- public BaseWeightedRouterPolicy() {
- }
-
- @Override
- public void reinitialize(FederationPolicyInitializationContext
- federationPolicyContext)
- throws FederationPolicyInitializationException {
- FederationPolicyInitializationContextValidator
- .validate(federationPolicyContext, this.getClass().getCanonicalName());
-
- // perform consistency checks
- WeightedPolicyInfo newPolicyInfo = WeightedPolicyInfo
- .fromByteBuffer(
- federationPolicyContext.getSubClusterPolicyConfiguration()
- .getParams());
-
- // if nothing has changed skip the rest of initialization
- if (policyInfo != null && policyInfo.equals(newPolicyInfo)) {
- return;
- }
-
- validate(newPolicyInfo);
- setPolicyInfo(newPolicyInfo);
- this.policyContext = federationPolicyContext;
- }
-
- /**
- * Overridable validation step for the policy configuration.
- * @param newPolicyInfo the configuration to test.
- * @throws FederationPolicyInitializationException if the configuration is
- * not valid.
- */
- public void validate(WeightedPolicyInfo newPolicyInfo) throws
- FederationPolicyInitializationException {
- if (newPolicyInfo == null) {
- throw new FederationPolicyInitializationException("The policy to "
- + "validate should not be null.");
- }
- Map<SubClusterIdInfo, Float> newWeights =
- newPolicyInfo.getRouterPolicyWeights();
- if (newWeights == null || newWeights.size() < 1) {
- throw new FederationPolicyInitializationException(
- "Weight vector cannot be null/empty.");
- }
- }
-
-
- /**
- * Getter method for the configuration weights.
- *
- * @return the {@link WeightedPolicyInfo} representing the policy
- * configuration.
- */
- public WeightedPolicyInfo getPolicyInfo() {
- return policyInfo;
- }
-
- /**
- * Setter method for the configuration weights.
- *
- * @param policyInfo the {@link WeightedPolicyInfo} representing the policy
- * configuration.
- */
- public void setPolicyInfo(
- WeightedPolicyInfo policyInfo) {
- this.policyInfo = policyInfo;
- }
-
- /**
- * Getter method for the {@link FederationPolicyInitializationContext}.
- * @return the context for this policy.
- */
- public FederationPolicyInitializationContext getPolicyContext() {
- return policyContext;
- }
-
- /**
- * Setter method for the {@link FederationPolicyInitializationContext}.
- * @param policyContext the context to assign to this policy.
- */
- public void setPolicyContext(
- FederationPolicyInitializationContext policyContext) {
- this.policyContext = policyContext;
- }
-
- /**
- * This methods gets active subclusters map from the {@code
- * FederationStateStoreFacade} and validate it not being null/empty.
- *
- * @return the map of ids to info for all active subclusters.
- * @throws YarnException if we can't get the list.
- */
- protected Map<SubClusterId, SubClusterInfo> getActiveSubclusters()
- throws YarnException {
-
- Map<SubClusterId, SubClusterInfo> activeSubclusters = getPolicyContext()
- .getFederationStateStoreFacade().getSubClusters(true);
-
- if (activeSubclusters == null || activeSubclusters.size() < 1) {
- throw new NoActiveSubclustersException(
- "Zero active subclusters, cannot pick where to send job.");
- }
- return activeSubclusters;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed1764cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
index 42c86cc..90ea0a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
@@ -35,11 +35,10 @@ public interface FederationRouterPolicy extends ConfigurableFederationPolicy {
* @param appSubmissionContext the context for the app being submitted.
*
* @return the sub-cluster as identified by {@link SubClusterId} to route the
- * request to.
+ * request to.
*
* @throws YarnException if the policy cannot determine a viable subcluster.
*/
SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException;
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-5300. Exclude generated
federation protobuf sources from YARN Javadoc/findbugs build
Posted by su...@apache.org.
YARN-5300. Exclude generated federation protobuf sources from YARN Javadoc/findbugs build
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4aca205b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4aca205b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4aca205b
Branch: refs/heads/YARN-2915
Commit: 4aca205bde41c526c9ef7fd8c2ecf7399fd02b03
Parents: 353a9b2
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Jul 19 15:08:25 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml | 3 +++
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aca205b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c090749..2f5451d 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -21,6 +21,9 @@
<Package name="org.apache.hadoop.yarn.proto" />
</Match>
<Match>
+ <Package name="org.apache.hadoop.yarn.federation.proto" />
+ </Match>
+ <Match>
<Class name="~org\.apache\.hadoop\.yarn\.ipc\.RpcProtos.*" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aca205b/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index c43588a..99b8b5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
- <excludePackageNames>org.apache.hadoop.yarn.proto</excludePackageNames>
+ <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto</excludePackageNames>
</configuration>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-5912. Fix breadcrumb issues
for various pages in new YARN UI. Contributed by Akhil P B.
Posted by su...@apache.org.
YARN-5912. Fix breadcrumb issues for various pages in new YARN UI. Contributed by Akhil P B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1fa084c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1fa084c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1fa084c4
Branch: refs/heads/YARN-2915
Commit: 1fa084c4254b89cd45210727ccb68725d583ff62
Parents: b9f8491
Author: Sunil G <su...@apache.org>
Authored: Tue Feb 14 22:29:21 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Feb 14 22:29:21 2017 +0530
----------------------------------------------------------------------
.../webapp/app/controllers/yarn-app-attempt.js | 2 +-
.../webapp/app/controllers/yarn-app-attempts.js | 2 +-
.../src/main/webapp/app/controllers/yarn-app.js | 2 +-
.../main/webapp/app/controllers/yarn-apps.js | 2 +-
.../app/controllers/yarn-container-log.js | 7 +++-
.../webapp/app/controllers/yarn-node-app.js | 7 +++-
.../webapp/app/controllers/yarn-node-apps.js | 2 +-
.../app/controllers/yarn-node-container.js | 39 ++++++++++++++++++++
.../app/controllers/yarn-node-containers.js | 2 +-
.../main/webapp/app/controllers/yarn-node.js | 2 +-
.../webapp/app/controllers/yarn-services.js | 2 +-
.../src/main/webapp/app/models/yarn-app.js | 2 +-
.../src/main/webapp/app/routes/yarn-node-app.js | 2 +-
.../webapp/app/routes/yarn-node-container.js | 2 +-
.../controllers/yarn-node-container-test.js | 30 +++++++++++++++
15 files changed, 90 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
index a458842..4c02361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js
@@ -27,7 +27,7 @@ export default Ember.Controller.extend({
routeName: 'application'
},{
text: "Applications",
- routeName: 'yarn-apps'
+ routeName: 'yarn-apps.apps'
}, {
text: `App [${appId}]`,
routeName: 'yarn-app',
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempts.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempts.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempts.js
index 9ebc2a6..92de2f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempts.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempts.js
@@ -27,7 +27,7 @@ export default Ember.Controller.extend({
routeName: 'application'
},{
text: "Applications",
- routeName: 'yarn-apps'
+ routeName: 'yarn-apps.apps'
}, {
text: `App [${appId}]`,
routeName: 'yarn-app',
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
index 309c895..9c1cb5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js
@@ -27,7 +27,7 @@ export default Ember.Controller.extend({
routeName: 'application'
},{
text: "Applications",
- routeName: 'yarn-apps'
+ routeName: 'yarn-apps.apps'
}, {
text: `App [${appId}]`,
routeName: 'yarn-app',
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps.js
index 396f83b..18bf682 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps.js
@@ -25,7 +25,7 @@ export default Ember.Controller.extend({
routeName: 'application'
}, {
text: "Applications",
- routeName: 'yarn-apps',
+ routeName: 'yarn-apps.apps',
}]
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
index 3352eaf..db7470d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
@@ -27,11 +27,14 @@ export default Ember.Controller.extend({
text: "Home",
routeName: 'application'
}, {
+ text: "Nodes",
+ routeName: 'yarn-nodes.table'
+ }, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `/#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
}, {
text: `Container [ ${containerInfo.id} ]`,
- href: `/#/yarn-node-container/${nodeInfo.id}/${nodeInfo.addr}/${containerInfo.id}`,
+ href: `#/yarn-node-container/${nodeInfo.id}/${nodeInfo.addr}/${containerInfo.id}`,
}, {
text: "Log",
}];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
index 76da281..3dc09fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
@@ -26,10 +26,13 @@ export default Ember.Controller.extend({
text: "Home",
routeName: 'application'
}, {
+ text: "Nodes",
+ routeName: 'yarn-nodes.table'
+ }, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `/#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
}, {
- text: "Application",
+ text: `Application [ ${nodeInfo.appId} ]`,
}];
})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
index 6e67ab0..6f53e74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
@@ -30,7 +30,7 @@ export default Ember.Controller.extend({
routeName: 'yarn-nodes.table'
}, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `/#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`
}, {
text: "Applications",
}];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
new file mode 100644
index 0000000..afcd518
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+
+export default Ember.Controller.extend({
+
+ breadcrumbs: Ember.computed("model.nodeInfo", function () {
+ var nodeInfo = this.get("model.nodeInfo");
+ return [{
+ text: "Home",
+ routeName: 'application'
+ },{
+ text: "Nodes",
+ routeName: 'yarn-nodes.table'
+ }, {
+ text: `Node [ ${nodeInfo.id} ]`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`
+ }, {
+ text: `Container [ ${nodeInfo.containerId} ]`
+ }];
+ })
+
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js
index abe4098..21d50a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-containers.js
@@ -30,7 +30,7 @@ export default Ember.Controller.extend({
routeName: 'yarn-nodes.table'
}, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `/#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`
}, {
text: "Containers",
}];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
index 0661415..55d9b3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node.js
@@ -30,7 +30,7 @@ export default Ember.Controller.extend({
routeName: 'yarn-nodes.table'
}, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `/#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
}];
})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-services.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-services.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-services.js
index 597962a..75bebf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-services.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-services.js
@@ -25,7 +25,7 @@ export default Ember.Controller.extend({
routeName: 'application'
}, {
text: "Applications",
- routeName: 'yarn-apps',
+ routeName: 'yarn-apps.apps',
}, {
text: "Long Running Services",
routeName: 'yarn-services',
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
index ea1334f..1366357 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-app.js
@@ -54,7 +54,7 @@ export default DS.Model.extend({
applicationExpiryTime: DS.attr('string'),
isFailed: function() {
- return this.get('finalStatus') == "FAILED";
+ return this.get('finalStatus') === "FAILED";
}.property("finalStatus"),
validatedFinishedTs: function() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
index 0a11930..8b2ad80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js
@@ -25,7 +25,7 @@ export default AbstractRoute.extend({
return Ember.RSVP.hash({
nodeApp: this.store.queryRecord('yarn-node-app',
{ nodeAddr : param.node_addr, appId: param.app_id }),
- nodeInfo: { id: param.node_id, addr: param.node_addr }
+ nodeInfo: { id: param.node_id, addr: param.node_addr, appId: param.app_id }
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-container.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-container.js
index b7a79de..5708fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-container.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-container.js
@@ -26,7 +26,7 @@ export default AbstractRoute.extend({
return Ember.RSVP.hash({
nodeContainer: this.store.queryRecord('yarn-node-container',
{ nodeHttpAddr: param.node_addr, containerId: param.container_id }),
- nodeInfo: { id: param.node_id, addr: param.node_addr }
+ nodeInfo: { id: param.node_id, addr: param.node_addr, containerId: param.container_id }
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fa084c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-node-container-test.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-node-container-test.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-node-container-test.js
new file mode 100644
index 0000000..c35e413
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/controllers/yarn-node-container-test.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleFor, test } from 'ember-qunit';
+
+moduleFor('controller:yarn-node-container', 'Unit | Controller | yarn node container', {
+ // Specify the other units that are required for this test.
+ // needs: ['controller:foo']
+});
+
+// Replace this with your real tests.
+test('it exists', function(assert) {
+ let controller = this.subject();
+ assert.ok(controller);
+});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-5676. Add a
HashBasedRouterPolicy,
and small policies and test refactoring. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5676. Add a HashBasedRouterPolicy, and small policies and test refactoring. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc338ed0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc338ed0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc338ed0
Branch: refs/heads/YARN-2915
Commit: fc338ed07ca40b7aba5591d8c2798b8b7a32e8b1
Parents: fbbeec4
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 22 15:02:22 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +-
.../policies/AbstractPolicyManager.java | 175 -----------------
.../policies/FederationPolicyManager.java | 117 ------------
.../PriorityBroadcastPolicyManager.java | 66 -------
.../federation/policies/RouterPolicyFacade.java | 1 +
.../policies/UniformBroadcastPolicyManager.java | 56 ------
.../policies/WeightedLocalityPolicyManager.java | 67 -------
.../policies/manager/AbstractPolicyManager.java | 190 +++++++++++++++++++
.../manager/FederationPolicyManager.java | 118 ++++++++++++
.../manager/HashBroadcastPolicyManager.java | 38 ++++
.../manager/PriorityBroadcastPolicyManager.java | 66 +++++++
.../manager/UniformBroadcastPolicyManager.java | 44 +++++
.../manager/WeightedLocalityPolicyManager.java | 67 +++++++
.../policies/manager/package-info.java | 19 ++
.../policies/router/AbstractRouterPolicy.java | 19 ++
.../policies/router/HashBasedRouterPolicy.java | 81 ++++++++
.../policies/router/LoadBasedRouterPolicy.java | 3 +
.../policies/router/PriorityRouterPolicy.java | 3 +
.../router/UniformRandomRouterPolicy.java | 10 +-
.../router/WeightedRandomRouterPolicy.java | 3 +
.../policies/BaseFederationPoliciesTest.java | 17 +-
.../policies/BasePolicyManagerTest.java | 108 -----------
...ionPolicyInitializationContextValidator.java | 1 +
.../TestPriorityBroadcastPolicyManager.java | 72 -------
.../policies/TestRouterPolicyFacade.java | 2 +
.../TestUniformBroadcastPolicyManager.java | 40 ----
.../TestWeightedLocalityPolicyManager.java | 79 --------
.../policies/manager/BasePolicyManagerTest.java | 104 ++++++++++
.../TestHashBasedBroadcastPolicyManager.java | 40 ++++
.../TestPriorityBroadcastPolicyManager.java | 72 +++++++
.../TestUniformBroadcastPolicyManager.java | 40 ++++
.../TestWeightedLocalityPolicyManager.java | 79 ++++++++
.../policies/router/BaseRouterPoliciesTest.java | 51 +++++
.../router/TestHashBasedRouterPolicy.java | 83 ++++++++
.../router/TestLoadBasedRouterPolicy.java | 3 +-
.../router/TestPriorityRouterPolicy.java | 3 +-
.../router/TestUniformRandomRouterPolicy.java | 3 +-
.../router/TestWeightedRandomRouterPolicy.java | 15 +-
38 files changed, 1160 insertions(+), 798 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index aca1177..1fbaa8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2526,7 +2526,8 @@ public class YarnConfiguration extends Configuration {
+ "policy-manager";
public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
- + ".hadoop.yarn.server.federation.policies.UniformBroadcastPolicyManager";
+ + ".hadoop.yarn.server.federation.policies"
+ + ".manager.UniformBroadcastPolicyManager";
public static final String FEDERATION_POLICY_MANAGER_PARAMS =
FEDERATION_PREFIX + "policy-manager-params";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
deleted file mode 100644
index e77f2e3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides basic implementation for common methods that multiple
- * policies will need to implement.
- */
-public abstract class AbstractPolicyManager implements
- FederationPolicyManager {
-
- private String queue;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class routerFederationPolicy;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class amrmProxyFederationPolicy;
-
- public static final Logger LOG =
- LoggerFactory.getLogger(AbstractPolicyManager.class);
- /**
- * This default implementation validates the
- * {@link FederationPolicyInitializationContext},
- * then checks whether it needs to reinstantiate the class (null or
- * mismatching type), and reinitialize the policy.
- *
- * @param federationPolicyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
- * instance
- *
- * @throws FederationPolicyInitializationException if the reinitalization is
- * not valid, and ensure
- * previous state is preserved
- */
- public FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext federationPolicyContext,
- FederationAMRMProxyPolicy oldInstance)
- throws FederationPolicyInitializationException {
-
- if (amrmProxyFederationPolicy == null) {
- throw new FederationPolicyInitializationException("The parameter "
- + "amrmProxyFederationPolicy should be initialized in "
- + this.getClass().getSimpleName() + " constructor.");
- }
-
- try {
- return (FederationAMRMProxyPolicy) internalPolicyGetter(
- federationPolicyContext, oldInstance, amrmProxyFederationPolicy);
- } catch (ClassCastException e) {
- throw new FederationPolicyInitializationException(e);
- }
-
- }
-
- /**
- * This default implementation validates the
- * {@link FederationPolicyInitializationContext},
- * then checks whether it needs to reinstantiate the class (null or
- * mismatching type), and reinitialize the policy.
- *
- * @param federationPolicyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return a valid and fully reinitalized {@link FederationRouterPolicy}
- * instance
- *
- * @throws FederationPolicyInitializationException if the reinitalization is
- * not valid, and ensure
- * previous state is preserved
- */
-
- public FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext federationPolicyContext,
- FederationRouterPolicy oldInstance)
- throws FederationPolicyInitializationException {
-
- //checks that sub-types properly initialize the types of policies
- if (routerFederationPolicy == null) {
- throw new FederationPolicyInitializationException("The policy "
- + "type should be initialized in " + this.getClass().getSimpleName()
- + " constructor.");
- }
-
- try {
- return (FederationRouterPolicy) internalPolicyGetter(
- federationPolicyContext, oldInstance, routerFederationPolicy);
- } catch (ClassCastException e) {
- throw new FederationPolicyInitializationException(e);
- }
- }
-
- @Override
- public String getQueue() {
- return queue;
- }
-
- @Override
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- /**
- * Common functionality to instantiate a reinitialize a {@link
- * ConfigurableFederationPolicy}.
- */
- private ConfigurableFederationPolicy internalPolicyGetter(
- final FederationPolicyInitializationContext federationPolicyContext,
- ConfigurableFederationPolicy oldInstance, Class policy)
- throws FederationPolicyInitializationException {
-
- FederationPolicyInitializationContextValidator
- .validate(federationPolicyContext, this.getClass().getCanonicalName());
-
- if (oldInstance == null || !oldInstance.getClass().equals(policy)) {
- try {
- oldInstance = (ConfigurableFederationPolicy) policy.newInstance();
- } catch (InstantiationException e) {
- throw new FederationPolicyInitializationException(e);
- } catch (IllegalAccessException e) {
- throw new FederationPolicyInitializationException(e);
- }
- }
-
- //copying the context to avoid side-effects
- FederationPolicyInitializationContext modifiedContext =
- updateContext(federationPolicyContext,
- oldInstance.getClass().getCanonicalName());
-
- oldInstance.reinitialize(modifiedContext);
- return oldInstance;
- }
-
- /**
- * This method is used to copy-on-write the context, that will be passed
- * downstream to the router/amrmproxy policies.
- */
- private FederationPolicyInitializationContext updateContext(
- FederationPolicyInitializationContext federationPolicyContext,
- String type) {
- // copying configuration and context to avoid modification of original
- SubClusterPolicyConfiguration newConf = SubClusterPolicyConfiguration
- .newInstance(federationPolicyContext
- .getSubClusterPolicyConfiguration());
- newConf.setType(type);
-
- return new FederationPolicyInitializationContext(newConf,
- federationPolicyContext.getFederationSubclusterResolver(),
- federationPolicyContext.getFederationStateStoreFacade(),
- federationPolicyContext.getHomeSubcluster());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
deleted file mode 100644
index 39fdba3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-/**
- *
- * Implementors need to provide the ability to serliaze a policy and its
- * configuration as a {@link SubClusterPolicyConfiguration}, as well as provide
- * (re)initialization mechanics for the underlying
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
- *
- * The serialization aspects are used by admin APIs or a policy engine to store
- * a serialized configuration in the {@code FederationStateStore}, while the
- * getters methods are used to obtain a propertly inizialized policy in the
- * {@code Router} and {@code AMRMProxy} respectively.
- *
- * This interface by design binds together {@link FederationAMRMProxyPolicy} and
- * {@link FederationRouterPolicy} and provide lifecycle support for
- * serialization and deserialization, to reduce configuration mistakes
- * (combining incompatible policies).
- *
- */
-public interface FederationPolicyManager {
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
- * current context, otherwise a new instance initialized with the current
- * context is provided. If the instance is compatible with the current class
- * the implementors should attempt to reinitalize (retaining state). To affect
- * a complete policy reset oldInstance should be null.
- *
- * @param policyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return an updated {@link FederationAMRMProxyPolicy }.
- *
- * @throws FederationPolicyInitializationException if the initialization
- * cannot be completed properly. The oldInstance should be still
- * valid in case of failed initialization.
- */
- FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext policyContext,
- FederationAMRMProxyPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationRouterPolicy} reinitialized with the current
- * context, otherwise a new instance initialized with the current context is
- * provided. If the instance is compatible with the current class the
- * implementors should attempt to reinitalize (retaining state). To affect a
- * complete policy reset oldInstance shoulb be set to null.
- *
- * @param policyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return an updated {@link FederationRouterPolicy}.
- *
- * @throws FederationPolicyInitializationException if the initalization cannot
- * be completed properly. The oldInstance should be still valid in
- * case of failed initialization.
- */
- FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext policyContext,
- FederationRouterPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
- /**
- * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
- * This is to be used when writing a policy object in the federation policy
- * store.
- *
- * @return a valid policy configuration representing this object
- * parametrization.
- *
- * @throws FederationPolicyInitializationException if the current state cannot
- * be serialized properly
- */
- SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException;
-
- /**
- * This method returns the queue this policy is configured for.
- *
- * @return the name of the queue.
- */
- String getQueue();
-
- /**
- * This methods provides a setter for the queue this policy is specified for.
- *
- * @param queue the name of the queue.
- */
- void setQueue(String queue);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
deleted file mode 100644
index ebdcf42..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Policy that allows operator to configure "weights" for routing. This picks a
- * {@link PriorityRouterPolicy} for the router and a
- * {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed to
- * work together.
- */
-public class PriorityBroadcastPolicyManager extends AbstractPolicyManager {
-
- private WeightedPolicyInfo weightedPolicyInfo;
-
- public PriorityBroadcastPolicyManager() {
- // this structurally hard-codes two compatible policies for Router and
- // AMRMProxy.
- routerFederationPolicy = PriorityRouterPolicy.class;
- amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
- weightedPolicyInfo = new WeightedPolicyInfo();
- }
-
- @Override
- public SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException {
- ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
- return SubClusterPolicyConfiguration.newInstance(getQueue(),
- this.getClass().getCanonicalName(), buf);
- }
-
- @VisibleForTesting
- public WeightedPolicyInfo getWeightedPolicyInfo() {
- return weightedPolicyInfo;
- }
-
- @VisibleForTesting
- public void setWeightedPolicyInfo(WeightedPolicyInfo weightedPolicyInfo) {
- this.weightedPolicyInfo = weightedPolicyInfo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
index a3fd15a..8c22623 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
deleted file mode 100644
index a01f8fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-import java.nio.ByteBuffer;
-
-/**
- * This class represents a simple implementation of a {@code
- * FederationPolicyManager}.
- *
- * It combines the basic policies: {@link UniformRandomRouterPolicy} and
- * {@link BroadcastAMRMProxyPolicy}, which are designed to work together and
- * "spread" the load among sub-clusters uniformly.
- *
- * This simple policy might impose heavy load on the RMs and return more
- * containers than a job requested as all requests are (replicated and)
- * broadcasted.
- */
-public class UniformBroadcastPolicyManager
- extends AbstractPolicyManager {
-
- public UniformBroadcastPolicyManager() {
- //this structurally hard-codes two compatible policies for Router and
- // AMRMProxy.
- routerFederationPolicy = UniformRandomRouterPolicy.class;
- amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
- }
-
- @Override
- public SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException {
- ByteBuffer buf = ByteBuffer.allocate(0);
- return SubClusterPolicyConfiguration
- .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
deleted file mode 100644
index f3c6673..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-import java.nio.ByteBuffer;
-
-/**
- * Policy that allows operator to configure "weights" for routing. This picks a
- * {@link WeightedRandomRouterPolicy} for the router and a {@link
- * LocalityMulticastAMRMProxyPolicy} for the amrmproxy as they are designed to
- * work together.
- */
-public class WeightedLocalityPolicyManager
- extends AbstractPolicyManager {
-
- private WeightedPolicyInfo weightedPolicyInfo;
-
- public WeightedLocalityPolicyManager() {
- //this structurally hard-codes two compatible policies for Router and
- // AMRMProxy.
- routerFederationPolicy = WeightedRandomRouterPolicy.class;
- amrmProxyFederationPolicy = LocalityMulticastAMRMProxyPolicy.class;
- weightedPolicyInfo = new WeightedPolicyInfo();
- }
-
- @Override
- public SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException {
- ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
- return SubClusterPolicyConfiguration
- .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
- }
-
- @VisibleForTesting
- public WeightedPolicyInfo getWeightedPolicyInfo() {
- return weightedPolicyInfo;
- }
-
- @VisibleForTesting
- public void setWeightedPolicyInfo(
- WeightedPolicyInfo weightedPolicyInfo) {
- this.weightedPolicyInfo = weightedPolicyInfo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java
new file mode 100644
index 0000000..f7a89c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class provides basic implementation for common methods that multiple
+ * policies will need to implement.
+ */
+public abstract class AbstractPolicyManager implements
+ FederationPolicyManager {
+
+ private String queue;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class routerFederationPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class amrmProxyFederationPolicy;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AbstractPolicyManager.class);
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ if (amrmProxyFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The parameter "
+ + "amrmProxyFederationPolicy should be initialized in "
+ + this.getClass().getSimpleName() + " constructor.");
+ }
+
+ try {
+ return (FederationAMRMProxyPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, amrmProxyFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+
+ }
+
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationRouterPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+
+ public FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ //checks that sub-types properly initialize the types of policies
+ if (routerFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The policy "
+ + "type should be initialized in " + this.getClass().getSimpleName()
+ + " constructor.");
+ }
+
+ try {
+ return (FederationRouterPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, routerFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ // default implementation works only for sub-classes which do not require
+ // any parameters
+ ByteBuffer buf = ByteBuffer.allocate(0);
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+
+ @Override
+ public String getQueue() {
+ return queue;
+ }
+
+ @Override
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ /**
+ * Common functionality to instantiate a reinitialize a {@link
+ * ConfigurableFederationPolicy}.
+ */
+ private ConfigurableFederationPolicy internalPolicyGetter(
+ final FederationPolicyInitializationContext federationPolicyContext,
+ ConfigurableFederationPolicy oldInstance, Class policy)
+ throws FederationPolicyInitializationException {
+
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ if (oldInstance == null || !oldInstance.getClass().equals(policy)) {
+ try {
+ oldInstance = (ConfigurableFederationPolicy) policy.newInstance();
+ } catch (InstantiationException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (IllegalAccessException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ //copying the context to avoid side-effects
+ FederationPolicyInitializationContext modifiedContext =
+ updateContext(federationPolicyContext,
+ oldInstance.getClass().getCanonicalName());
+
+ oldInstance.reinitialize(modifiedContext);
+ return oldInstance;
+ }
+
+ /**
+ * This method is used to copy-on-write the context, that will be passed
+ * downstream to the router/amrmproxy policies.
+ */
+ private FederationPolicyInitializationContext updateContext(
+ FederationPolicyInitializationContext federationPolicyContext,
+ String type) {
+ // copying configuration and context to avoid modification of original
+ SubClusterPolicyConfiguration newConf = SubClusterPolicyConfiguration
+ .newInstance(federationPolicyContext
+ .getSubClusterPolicyConfiguration());
+ newConf.setType(type);
+
+ return new FederationPolicyInitializationContext(newConf,
+ federationPolicyContext.getFederationSubclusterResolver(),
+ federationPolicyContext.getFederationStateStoreFacade(),
+ federationPolicyContext.getHomeSubcluster());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java
new file mode 100644
index 0000000..1434c80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+/**
+ *
+ * Implementors need to provide the ability to serliaze a policy and its
+ * configuration as a {@link SubClusterPolicyConfiguration}, as well as provide
+ * (re)initialization mechanics for the underlying
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
+ *
+ * The serialization aspects are used by admin APIs or a policy engine to store
+ * a serialized configuration in the {@code FederationStateStore}, while the
+ * getters methods are used to obtain a propertly inizialized policy in the
+ * {@code Router} and {@code AMRMProxy} respectively.
+ *
+ * This interface by design binds together {@link FederationAMRMProxyPolicy} and
+ * {@link FederationRouterPolicy} and provide lifecycle support for
+ * serialization and deserialization, to reduce configuration mistakes
+ * (combining incompatible policies).
+ *
+ */
+public interface FederationPolicyManager {
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
+ * current context, otherwise a new instance initialized with the current
+ * context is provided. If the instance is compatible with the current class
+ * the implementors should attempt to reinitalize (retaining state). To affect
+ * a complete policy reset oldInstance should be null.
+ *
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return an updated {@link FederationAMRMProxyPolicy }.
+ *
+ * @throws FederationPolicyInitializationException if the initialization
+ * cannot be completed properly. The oldInstance should be still
+ * valid in case of failed initialization.
+ */
+ FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext policyContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationRouterPolicy} reinitialized with the current
+ * context, otherwise a new instance initialized with the current context is
+ * provided. If the instance is compatible with the current class the
+ * implementors should attempt to reinitalize (retaining state). To affect a
+ * complete policy reset oldInstance shoulb be set to null.
+ *
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return an updated {@link FederationRouterPolicy}.
+ *
+ * @throws FederationPolicyInitializationException if the initalization cannot
+ * be completed properly. The oldInstance should be still valid in
+ * case of failed initialization.
+ */
+ FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext policyContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
+ * This is to be used when writing a policy object in the federation policy
+ * store.
+ *
+ * @return a valid policy configuration representing this object
+ * parametrization.
+ *
+ * @throws FederationPolicyInitializationException if the current state cannot
+ * be serialized properly
+ */
+ SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException;
+
+ /**
+ * This method returns the queue this policy is configured for.
+ *
+ * @return the name of the queue.
+ */
+ String getQueue();
+
+ /**
+ * This methods provides a setter for the queue this policy is specified for.
+ *
+ * @param queue the name of the queue.
+ */
+ void setQueue(String queue);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java
new file mode 100644
index 0000000..08ab08f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.HashBasedRouterPolicy;
+
+/**
+ * Policy that routes applications via hashing of their queuename, and broadcast
+ * resource requests. This picks a {@link HashBasedRouterPolicy} for the router
+ * and a {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed
+ * to work together.
+ */
+public class HashBroadcastPolicyManager extends AbstractPolicyManager {
+
+ public HashBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = HashBasedRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..8139e12
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link PriorityRouterPolicy} for the router and a
+ * {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class PriorityBroadcastPolicyManager extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public PriorityBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = PriorityRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration.newInstance(getQueue(),
+ this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..5db0466
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+
+/**
+ * This class represents a simple implementation of a {@code
+ * FederationPolicyManager}.
+ *
+ * It combines the basic policies: {@link UniformRandomRouterPolicy} and
+ * {@link BroadcastAMRMProxyPolicy}, which are designed to work together and
+ * "spread" the load among sub-clusters uniformly.
+ *
+ * This simple policy might impose heavy load on the RMs and return more
+ * containers than a job requested as all requests are (replicated and)
+ * broadcasted.
+ */
+public class UniformBroadcastPolicyManager extends AbstractPolicyManager {
+
+ public UniformBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = UniformRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..109b534
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link WeightedRandomRouterPolicy} for the router and a {@link
+ * LocalityMulticastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class WeightedLocalityPolicyManager
+ extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public WeightedLocalityPolicyManager() {
+ //this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = WeightedRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(
+ WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java
new file mode 100644
index 0000000..9515c01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Various implementation of FederationPolicyManager. **/
+package org.apache.hadoop.yarn.server.federation.policies.manager;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
index f49af1d..730fb41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.federation.policies.router;
import java.util.Map;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -44,4 +47,20 @@ public abstract class AbstractRouterPolicy extends
}
}
+ public void validate(ApplicationSubmissionContext appSubmissionContext)
+ throws FederationPolicyException {
+
+ if (appSubmissionContext == null) {
+ throw new FederationPolicyException(
+ "Cannot route an application with null context.");
+ }
+
+ // if the queue is not specified we set it to default value, to be
+ // compatible with YARN behavior.
+ String queue = appSubmissionContext.getQueue();
+ if (queue == null) {
+ appSubmissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
new file mode 100644
index 0000000..e40e87e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * This {@link FederationRouterPolicy} pick a subcluster based on the hash of
+ * the job's queue name. Useful to provide a default behavior when too many
+ * queues exist in a system. This also ensures that all jobs belonging to a
+ * queue are mapped to the same sub-cluster (likely help with locality).
+ */
+public class HashBasedRouterPolicy extends AbstractRouterPolicy {
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ // note: this overrides BaseRouterPolicy and ignores the weights
+ setPolicyContext(federationPolicyContext);
+ }
+
+ /**
+ * Simply picks from alphabetically-sorted active subclusters based on the
+ * hash of quey name. Jobs of the same queue will all be routed to the same
+ * sub-cluster, as far as the number of active sub-cluster and their names
+ * remain the same.
+ *
+ * @param appSubmissionContext the context for the app being submitted.
+ *
+ * @return a hash-based chosen subcluster.
+ *
+ * @throws YarnException if there are no active subclusters.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+
+ // throws if no active subclusters available
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ validate(appSubmissionContext);
+
+ int chosenPosition = Math.abs(
+ appSubmissionContext.getQueue().hashCode() % activeSubclusters.size());
+
+ List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
+ Collections.sort(list);
+ return list.get(chosenPosition);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 5de749f..2ca15bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -64,6 +64,9 @@ public class LoadBasedRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
index bc3a1f7..13d9140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -36,6 +36,9 @@ public class PriorityRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
index b8f9cc3..d820449 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -48,11 +48,10 @@ public class UniformRandomRouterPolicy extends AbstractRouterPolicy {
}
@Override
- public void reinitialize(
- FederationPolicyInitializationContext policyContext)
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
- FederationPolicyInitializationContextValidator
- .validate(policyContext, this.getClass().getCanonicalName());
+ FederationPolicyInitializationContextValidator.validate(policyContext,
+ this.getClass().getCanonicalName());
// note: this overrides AbstractRouterPolicy and ignores the weights
@@ -73,6 +72,9 @@ public class UniformRandomRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
index ac75ae9..5727134 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -43,6 +43,9 @@ public class WeightedRandomRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
index ba897da..6bd8bf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.federation.policies;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -35,8 +36,10 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPo
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
import org.junit.Test;
@@ -46,7 +49,7 @@ import org.junit.Test;
public abstract class BaseFederationPoliciesTest {
private ConfigurableFederationPolicy policy;
- private WeightedPolicyInfo policyInfo;
+ private WeightedPolicyInfo policyInfo = mock(WeightedPolicyInfo.class);
private Map<SubClusterId, SubClusterInfo> activeSubclusters = new HashMap<>();
private FederationPolicyInitializationContext federationPolicyContext;
private ApplicationSubmissionContext applicationSubmissionContext =
@@ -103,7 +106,7 @@ public abstract class BaseFederationPoliciesTest {
((FederationRouterPolicy) localPolicy)
.getHomeSubcluster(getApplicationSubmissionContext());
} else {
- String[] hosts = new String[] {"host1", "host2" };
+ String[] hosts = new String[] {"host1", "host2"};
List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
.createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
((FederationAMRMProxyPolicy) localPolicy)
@@ -170,4 +173,14 @@ public abstract class BaseFederationPoliciesTest {
this.homeSubCluster = homeSubCluster;
}
+ public void setMockActiveSubclusters(int numSubclusters) {
+ for (int i = 1; i <= numSubclusters; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
deleted file mode 100644
index c609886..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This class provides common test methods for testing {@code
- * FederationPolicyManager}s.
- */
-public abstract class BasePolicyManagerTest {
-
-
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected FederationPolicyManager wfp = null;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class expectedPolicyManager;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class expectedAMRMProxyPolicy;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class expectedRouterPolicy;
-
-
- @Test
- public void testSerializeAndInstantiate() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy,
- expectedRouterPolicy);
- }
-
- @Test(expected = FederationPolicyInitializationException.class)
- public void testSerializeAndInstantiateBad1() throws Exception {
- serializeAndDeserializePolicyManager(wfp, String.class,
- expectedAMRMProxyPolicy, expectedRouterPolicy);
- }
-
- @Test(expected = AssertionError.class)
- public void testSerializeAndInstantiateBad2() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- String.class, expectedRouterPolicy);
- }
-
- @Test(expected = AssertionError.class)
- public void testSerializeAndInstantiateBad3() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy, String.class);
- }
-
- protected static void serializeAndDeserializePolicyManager(
- FederationPolicyManager wfp, Class policyManagerType,
- Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception {
-
- // serializeConf it in a context
- SubClusterPolicyConfiguration fpc =
- wfp.serializeConf();
- fpc.setType(policyManagerType.getCanonicalName());
- FederationPolicyInitializationContext context = new
- FederationPolicyInitializationContext();
- context.setSubClusterPolicyConfiguration(fpc);
- context
- .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
- context.setFederationSubclusterResolver(
- FederationPoliciesTestUtil.initResolver());
- context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster"));
-
- // based on the "context" created instantiate new class and use it
- Class c = Class.forName(wfp.getClass().getCanonicalName());
- FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance();
-
- FederationAMRMProxyPolicy federationAMRMProxyPolicy =
- wfp2.getAMRMPolicy(context, null);
-
- //needed only for tests (getARMRMPolicy change the "type" in conf)
- fpc.setType(wfp.getClass().getCanonicalName());
-
- FederationRouterPolicy federationRouterPolicy =
- wfp2.getRouterPolicy(context, null);
-
- Assert.assertEquals(federationAMRMProxyPolicy.getClass(),
- expAMRMProxyPolicy);
-
- Assert.assertEquals(federationRouterPolicy.getClass(),
- expRouterPolicy);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index d906b92..611a486 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc338ed0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
deleted file mode 100644
index 5e5bc83..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Simple test of {@link PriorityBroadcastPolicyManager}.
- */
-public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest {
-
- private WeightedPolicyInfo policyInfo;
-
- @Before
- public void setup() {
- // configure a policy
-
- wfp = new PriorityBroadcastPolicyManager();
- wfp.setQueue("queue1");
- SubClusterId sc1 = SubClusterId.newInstance("sc1");
- SubClusterId sc2 = SubClusterId.newInstance("sc2");
- policyInfo = new WeightedPolicyInfo();
-
- Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
- routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
- routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
- policyInfo.setRouterPolicyWeights(routerWeights);
-
- ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo);
-
- // set expected params that the base test class will use for tests
- expectedPolicyManager = PriorityBroadcastPolicyManager.class;
- expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
- expectedRouterPolicy = PriorityRouterPolicy.class;
- }
-
- @Test
- public void testPolicyInfoSetCorrectly() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy, expectedRouterPolicy);
-
- // check the policyInfo propagates through ser/der correctly
- Assert.assertEquals(
- ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(),
- policyInfo);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-6113. Re-direct NM Web
Service to get container logs for finished applications. Contributed by Xuan
Gong.
Posted by su...@apache.org.
YARN-6113. Re-direct NM Web Service to get container logs for finished applications. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/464ff479
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/464ff479
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/464ff479
Branch: refs/heads/YARN-2915
Commit: 464ff479ceec76609dca3539081de6b503b17325
Parents: 243c0f3
Author: Junping Du <ju...@apache.org>
Authored: Mon Feb 13 06:12:54 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Mon Feb 13 06:12:54 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 31 ++++++-
.../src/main/resources/yarn-default.xml | 8 ++
.../nodemanager/webapp/NMWebServices.java | 45 +++++++++-
.../nodemanager/webapp/TestNMWebServices.java | 93 ++++++++++++++++++--
5 files changed, 167 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/464ff479/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7887fbc..136227a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1077,7 +1077,10 @@ public class YarnConfiguration extends Configuration {
public static final String YARN_LOG_SERVER_URL =
YARN_PREFIX + "log.server.url";
-
+
+ public static final String YARN_LOG_SERVER_WEBSERVICE_URL =
+ YARN_PREFIX + "log.server.web-service.url";
+
public static final String YARN_TRACKING_URL_GENERATOR =
YARN_PREFIX + "tracking.url.generator";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/464ff479/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 89f0551..e412173 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -26,6 +26,7 @@ import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -428,7 +429,8 @@ public class WebAppUtils {
return Arrays.asList("text", "octet-stream");
}
- private static String getURLEncodedQueryString(HttpServletRequest request) {
+ private static String getURLEncodedQueryString(HttpServletRequest request,
+ String parameterToRemove) {
String queryString = request.getQueryString();
if (queryString != null && !queryString.isEmpty()) {
String reqEncoding = request.getCharacterEncoding();
@@ -436,20 +438,41 @@ public class WebAppUtils {
reqEncoding = "ISO-8859-1";
}
Charset encoding = Charset.forName(reqEncoding);
- List<NameValuePair> params = URLEncodedUtils.parse(queryString, encoding);
+ List<NameValuePair> params = URLEncodedUtils.parse(queryString,
+ encoding);
+ if (parameterToRemove != null && !parameterToRemove.isEmpty()) {
+ Iterator<NameValuePair> paramIterator = params.iterator();
+ while(paramIterator.hasNext()) {
+ NameValuePair current = paramIterator.next();
+ if (current.getName().equals(parameterToRemove)) {
+ paramIterator.remove();
+ }
+ }
+ }
return URLEncodedUtils.format(params, encoding);
}
return null;
}
/**
+ * Get a query string which removes the passed parameter.
+ * @param httpRequest HttpServletRequest with the request details
+ * @param parameterName the query parameters must be removed
+ * @return the query parameter string
+ */
+ public static String removeQueryParams(HttpServletRequest httpRequest,
+ String parameterName) {
+ return getURLEncodedQueryString(httpRequest, parameterName);
+ }
+
+ /**
* Get a HTML escaped uri with the query parameters of the request.
* @param request HttpServletRequest with the request details
* @return HTML escaped uri with the query paramters
*/
public static String getHtmlEscapedURIWithQueryString(
HttpServletRequest request) {
- String urlEncodedQueryString = getURLEncodedQueryString(request);
+ String urlEncodedQueryString = getURLEncodedQueryString(request, null);
if (urlEncodedQueryString != null) {
return HtmlQuoting.quoteHtmlChars(
request.getRequestURI() + "?" + urlEncodedQueryString);
@@ -466,7 +489,7 @@ public class WebAppUtils {
public static String appendQueryParams(HttpServletRequest request,
String targetUri) {
String ret = targetUri;
- String urlEncodedQueryString = getURLEncodedQueryString(request);
+ String urlEncodedQueryString = getURLEncodedQueryString(request, null);
if (urlEncodedQueryString != null) {
ret += "?" + urlEncodedQueryString;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/464ff479/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1e929a8..4ca46f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2650,6 +2650,14 @@
<property>
<description>
+ URL for log aggregation server web service
+ </description>
+ <name>yarn.log.server.web-service.url</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
RM Application Tracking URL
</description>
<name>yarn.tracking.url.generator</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/464ff479/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index a59e010..44b232d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -87,6 +88,7 @@ public class NMWebServices {
private WebApp webapp;
private static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
+ private final String redirectWSUrl;
private @javax.ws.rs.core.Context
HttpServletRequest request;
@@ -103,6 +105,8 @@ public class NMWebServices {
this.nmContext = nm;
this.rview = view;
this.webapp = webapp;
+ this.redirectWSUrl = this.nmContext.getConf().get(
+ YarnConfiguration.YARN_LOG_SERVER_WEBSERVICE_URL);
}
private void init() {
@@ -270,6 +274,9 @@ public class NMWebServices {
} catch (IOException ex) {
// Something wrong with we tries to access the remote fs for the logs.
// Skip it and do nothing
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(ex.getMessage());
+ }
}
GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<
ContainerLogsInfo>>(containersLogsInfo){};
@@ -280,7 +287,13 @@ public class NMWebServices {
resp.header("X-Content-Type-Options", "nosniff");
return resp.build();
} catch (Exception ex) {
- throw new WebApplicationException(ex);
+ if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
+ throw new WebApplicationException(ex);
+ }
+ // redirect the request to the configured log server
+ String redirectURI = "/containers/" + containerIdStr
+ + "/logs";
+ return createRedirectResponse(hsr, redirectWSUrl, redirectURI);
}
}
@@ -377,7 +390,14 @@ public class NMWebServices {
logFile = ContainerLogsUtils.getContainerLogFile(
containerId, filename, request.getRemoteUser(), nmContext);
} catch (NotFoundException ex) {
- return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build();
+ if (redirectWSUrl == null || redirectWSUrl.isEmpty()) {
+ return Response.status(Status.NOT_FOUND).entity(ex.getMessage())
+ .build();
+ }
+ // redirect the request to the configured log server
+ String redirectURI = "/containers/" + containerIdStr
+ + "/logs/" + filename;
+ return createRedirectResponse(request, redirectWSUrl, redirectURI);
} catch (YarnException ex) {
return Response.serverError().entity(ex.getMessage()).build();
}
@@ -464,4 +484,25 @@ public class NMWebServices {
}
return Long.parseLong(bytes);
}
+
+ private Response createRedirectResponse(HttpServletRequest httpRequest,
+ String redirectWSUrlPrefix, String uri) {
+ // redirect the request to the configured log server
+ StringBuilder redirectPath = new StringBuilder();
+ if (redirectWSUrlPrefix.endsWith("/")) {
+ redirectWSUrlPrefix = redirectWSUrlPrefix.substring(0,
+ redirectWSUrlPrefix.length() - 1);
+ }
+ redirectPath.append(redirectWSUrlPrefix + uri);
+ // append all the request query parameters except nodeId parameter
+ String requestParams = WebAppUtils.removeQueryParams(httpRequest,
+ YarnWebServiceParams.NM_ID);
+ if (requestParams != null && !requestParams.isEmpty()) {
+ redirectPath.append("?" + requestParams);
+ }
+ ResponseBuilder res = Response.status(
+ HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ res.header("Location", redirectPath.toString());
+ return res.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/464ff479/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 7764ceb..e3773d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -27,7 +28,11 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
import java.util.List;
+import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -59,6 +64,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -97,6 +103,7 @@ public class TestNMWebServices extends JerseyTestBase {
private static ApplicationACLsManager aclsManager;
private static LocalDirsHandlerService dirsHandler;
private static WebApp nmWebApp;
+ private static final String LOGSERVICEWSADDR = "test:1234";
private static final File testRootDir = new File("target",
TestNMWebServices.class.getSimpleName());
@@ -115,6 +122,8 @@ public class TestNMWebServices extends JerseyTestBase {
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
testRemoteLogDir.getAbsolutePath());
+ conf.set(YarnConfiguration.YARN_LOG_SERVER_WEBSERVICE_URL,
+ LOGSERVICEWSADDR);
dirsHandler = new LocalDirsHandlerService();
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
@@ -351,6 +360,58 @@ public class TestNMWebServices extends JerseyTestBase {
testContainerLogs(r, containerId);
}
+ @Test (timeout = 10000)
+ public void testNMRedirect() {
+ ApplicationId noExistAppId = ApplicationId.newInstance(
+ System.currentTimeMillis(), 2000);
+ ApplicationAttemptId noExistAttemptId = ApplicationAttemptId.newInstance(
+ noExistAppId, 150);
+ ContainerId noExistContainerId = ContainerId.newContainerId(
+ noExistAttemptId, 250);
+ String fileName = "syslog";
+ WebResource r = resource();
+
+ // check the old api
+ URI requestURI = r.path("ws").path("v1").path("node")
+ .path("containerlogs").path(noExistContainerId.toString())
+ .path(fileName).queryParam("user.name", "user")
+ .queryParam(YarnWebServiceParams.NM_ID, "localhost:1111")
+ .getURI();
+ String redirectURL = getRedirectURL(requestURI.toString());
+ assertTrue(redirectURL != null);
+ assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
+ assertTrue(redirectURL.contains(noExistContainerId.toString()));
+ assertTrue(redirectURL.contains("/logs/" + fileName));
+ assertTrue(redirectURL.contains("user.name=" + "user"));
+ assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID));
+
+ // check the new api
+ requestURI = r.path("ws").path("v1").path("node")
+ .path("containers").path(noExistContainerId.toString())
+ .path("logs").path(fileName).queryParam("user.name", "user")
+ .queryParam(YarnWebServiceParams.NM_ID, "localhost:1111")
+ .getURI();
+ redirectURL = getRedirectURL(requestURI.toString());
+ assertTrue(redirectURL != null);
+ assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
+ assertTrue(redirectURL.contains(noExistContainerId.toString()));
+ assertTrue(redirectURL.contains("/logs/" + fileName));
+ assertTrue(redirectURL.contains("user.name=" + "user"));
+ assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID));
+
+ requestURI = r.path("ws").path("v1").path("node")
+ .path("containers").path(noExistContainerId.toString())
+ .path("logs").queryParam("user.name", "user")
+ .queryParam(YarnWebServiceParams.NM_ID, "localhost:1111")
+ .getURI();
+ redirectURL = getRedirectURL(requestURI.toString());
+ assertTrue(redirectURL != null);
+ assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
+ assertTrue(redirectURL.contains(noExistContainerId.toString()));
+ assertTrue(redirectURL.contains("user.name=" + "user"));
+ assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID));
+ }
+
private void testContainerLogs(WebResource r, ContainerId containerId)
throws IOException {
final String containerIdStr = containerId.toString();
@@ -451,13 +512,12 @@ public class TestNMWebServices extends JerseyTestBase {
+ WebAppUtils.listSupportedLogContentType(), responseText);
assertEquals(400, response.getStatus());
- // ask for file that doesn't exist
- response = r.path("uhhh")
- .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
- assertEquals(Status.NOT_FOUND.getStatusCode(),
- response.getStatus());
- responseText = response.getEntity(String.class);
- assertTrue(responseText.contains("Cannot find this log on the local disk."));
+ // ask for file that doesn't exist and it will re-direct to
+ // the log server
+ URI requestURI = r.path("uhhh").getURI();
+ String redirectURL = getRedirectURL(requestURI.toString());
+ assertTrue(redirectURL != null);
+ assertTrue(redirectURL.contains(LOGSERVICEWSADDR));
// Get container log files' name
WebResource r1 = resource();
@@ -630,4 +690,21 @@ public class TestNMWebServices extends JerseyTestBase {
int postfixIndex = fullMessage.indexOf(postfix);
return fullMessage.substring(prefixIndex, postfixIndex);
}
-}
+
+ private static String getRedirectURL(String url) {
+ String redirectUrl = null;
+ try {
+ HttpURLConnection conn = (HttpURLConnection) new URL(url)
+ .openConnection();
+ // do not automatically follow the redirection
+ // otherwise we get too many redirections exception
+ conn.setInstanceFollowRedirects(false);
+ if(conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) {
+ redirectUrl = conn.getHeaderField("Location");
+ }
+ } catch (Exception e) {
+ // throw new RuntimeException(e);
+ }
+ return redirectUrl;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-6061. Addendum. Remove
extraneous change.
Posted by su...@apache.org.
YARN-6061. Addendum. Remove extraneous change.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/353a9b2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/353a9b2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/353a9b2d
Branch: refs/heads/YARN-2915
Commit: 353a9b2d9165a221491395edbadf8acc3a39990b
Parents: 8e53f2b
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Tue Feb 14 15:19:52 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Tue Feb 14 15:19:52 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/policies/DominantResourceFairnessPolicy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/353a9b2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 7a29735..ad41b11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -90,7 +90,7 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
@Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
- return Resources.greaterThan(CALCULATOR, null, usage, fairShare);
+ return !Resources.fitsIn(usage, fairShare);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-5307. Federation Application
State Store internal APIs
Posted by su...@apache.org.
YARN-5307. Federation Application State Store internal APIs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c503284
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c503284
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c503284
Branch: refs/heads/YARN-2915
Commit: 4c50328448666fb1192b0af4447dcd587d541425
Parents: 9e5e94b
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Aug 5 11:52:44 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
...ederationApplicationHomeSubClusterStore.java | 126 ++++++++++++
.../AddApplicationHomeSubClusterRequest.java | 72 +++++++
.../AddApplicationHomeSubClusterResponse.java | 44 +++++
.../records/ApplicationHomeSubCluster.java | 124 ++++++++++++
.../DeleteApplicationHomeSubClusterRequest.java | 65 +++++++
...DeleteApplicationHomeSubClusterResponse.java | 43 +++++
.../GetApplicationHomeSubClusterRequest.java | 64 +++++++
.../GetApplicationHomeSubClusterResponse.java | 73 +++++++
.../GetApplicationsHomeSubClusterRequest.java | 40 ++++
.../GetApplicationsHomeSubClusterResponse.java | 75 ++++++++
.../UpdateApplicationHomeSubClusterRequest.java | 74 ++++++++
...UpdateApplicationHomeSubClusterResponse.java | 43 +++++
...dApplicationHomeSubClusterRequestPBImpl.java | 132 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 78 ++++++++
.../pb/ApplicationHomeSubClusterPBImpl.java | 167 ++++++++++++++++
...eApplicationHomeSubClusterRequestPBImpl.java | 130 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 78 ++++++++
...tApplicationHomeSubClusterRequestPBImpl.java | 135 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 132 +++++++++++++
...ApplicationsHomeSubClusterRequestPBImpl.java | 78 ++++++++
...pplicationsHomeSubClusterResponsePBImpl.java | 190 +++++++++++++++++++
.../pb/GetSubClustersInfoResponsePBImpl.java | 6 +-
...eApplicationHomeSubClusterRequestPBImpl.java | 132 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 78 ++++++++
.../proto/yarn_server_federation_protos.proto | 45 ++++-
.../records/TestFederationProtocolRecords.java | 81 ++++++++
26 files changed, 2301 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
new file mode 100644
index 0000000..217ee2e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+
+/**
+ * FederationApplicationHomeSubClusterStore maintains the state of all
+ * <em>Applications</em> that have been submitted to the federated cluster.
+ *
+ * *
+ * <p>
+ * The mapping details contains:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ *
+ */
+@Private
+@Unstable
+public interface FederationApplicationHomeSubClusterStore {
+
+ /**
+ * Get the {@link Version} of the underlying federation application state
+ * store.
+ *
+ * @return the {@link Version} of the underlying federation application state
+ * store
+ */
+ Version getApplicationStateStoreVersion();
+
+ /**
+ * Register the home {@code SubClusterId} of the newly submitted
+ * {@code ApplicationId}. Currently response is empty if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @param request the request to register a new application with its home
+ * sub-cluster
+ * @return empty on successful registration of the application in the
+ * StateStore, if not an exception reporting reason for a failure
+ * @throws YarnException if the request is invalid/fails
+ */
+ AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Update the home {@code SubClusterId} of a previously submitted
+ * {@code ApplicationId}. Currently response is empty if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @param request the request to update the home sub-cluster of an
+ * application.
+ * @return empty on successful update of the application in the StateStore, if
+ * not an exception reporting reason for a failure
+ * @throws YarnException if the request is invalid/fails
+ */
+ UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Get information about the application identified by the input
+ * {@code ApplicationId}.
+ *
+ * @param request contains the application queried
+ * @return {@code ApplicationHomeSubCluster} containing the application's
+ * home subcluster
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Get the {@code ApplicationHomeSubCluster} list representing the mapping
+ * of all submitted applications to it's home sub-cluster.
+ *
+ * @param request empty representing all applications
+ * @return the mapping of all submitted application to it's home sub-cluster
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Delete the mapping of home {@code SubClusterId} of a previously submitted
+ * {@code ApplicationId}. Currently response is empty if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @param request the request to delete the home sub-cluster of an
+ * application.
+ * @return empty on successful update of the application in the StateStore, if
+ * not an exception reporting reason for a failure
+ * @throws YarnException if the request is invalid/fails
+ */
+ DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ DeleteApplicationHomeSubClusterRequest request) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..9cb0589
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent by the <code>Router</code> to <code>Federation state
+ * store</code> to map the home subcluster of a newly submitted application.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class AddApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static AddApplicationHomeSubClusterRequest newInstance(
+ ApplicationHomeSubCluster applicationHomeSubCluster) {
+ AddApplicationHomeSubClusterRequest mapRequest =
+ Records.newRecord(AddApplicationHomeSubClusterRequest.class);
+ mapRequest.setApplicationHomeSubCluster(applicationHomeSubCluster);
+ return mapRequest;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @return the mapping of the application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationHomeSubCluster getApplicationHomeSubCluster();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @param applicationHomeSubCluster the mapping of the application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..2145dd1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * AddApplicationHomeSubClusterResponse contains the answer from the
+ * {@code FederationApplicationHomeSubClusterStore} to a request to insert a
+ * newly generated applicationId and its owner. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ *
+ */
+@Private
+@Unstable
+public abstract class AddApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static AddApplicationHomeSubClusterResponse newInstance() {
+ AddApplicationHomeSubClusterResponse response =
+ Records.newRecord(AddApplicationHomeSubClusterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
new file mode 100644
index 0000000..5e4c7cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * ApplicationHomeSubCluster is a report of the runtime information of the
+ * application that is running in the federated cluster.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link ApplicationId}</li>
+ * <li>{@link SubClusterId}</li>
+ * </ul>
+ *
+ */
+@Private
+@Unstable
+public abstract class ApplicationHomeSubCluster {
+
+ @Private
+ @Unstable
+ public static ApplicationHomeSubCluster newInstance(ApplicationId appId,
+ SubClusterId homeSubCluster) {
+ ApplicationHomeSubCluster appMapping =
+ Records.newRecord(ApplicationHomeSubCluster.class);
+ appMapping.setApplicationId(appId);
+ appMapping.setHomeSubCluster(homeSubCluster);
+ return appMapping;
+ }
+
+ /**
+ * Get the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @return the application identifier
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ /**
+ * Set the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @param applicationId the application identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the home
+ * subcluster in which the ApplicationMaster of the application is running.
+ *
+ * @return the home subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getHomeSubCluster();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the home
+ * subcluster in which the ApplicationMaster of the application is running.
+ *
+ * @param homeSubCluster the home subcluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setHomeSubCluster(SubClusterId homeSubCluster);
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ApplicationHomeSubCluster other = (ApplicationHomeSubCluster) obj;
+ if (!this.getApplicationId().equals(other.getApplicationId())) {
+ return false;
+ }
+ return this.getHomeSubCluster().equals(other.getHomeSubCluster());
+ }
+
+ @Override
+ public int hashCode() {
+ return getApplicationId().hashCode() * 31 + getHomeSubCluster().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "ApplicationHomeSubCluster [getApplicationId()="
+ + getApplicationId() + ", getHomeSubCluster()=" + getHomeSubCluster()
+ + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..f678aee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The request to <code>Federation state store</code> to delete the mapping of
+ * home subcluster of a submitted application.
+ */
+@Private
+@Unstable
+public abstract class DeleteApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static DeleteApplicationHomeSubClusterRequest newInstance(
+ ApplicationId applicationId) {
+ DeleteApplicationHomeSubClusterRequest deleteApplicationRequest =
+ Records.newRecord(DeleteApplicationHomeSubClusterRequest.class);
+ deleteApplicationRequest.setApplicationId(applicationId);
+ return deleteApplicationRequest;
+ }
+
+ /**
+ * Get the identifier of the {@link ApplicationId} to be removed from
+ * <code>Federation state store</code> .
+ *
+ * @return the identifier of the application to be removed from Federation
+ * State Store.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ /**
+ * Set the identifier of the {@link ApplicationId} to be removed from
+ * <code>Federation state store</code> .
+ *
+ * @param applicationId the identifier of the application to be removed from
+ * Federation State Store.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..fb1bef9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * DeleteApplicationHomeSubClusterResponse contains the answer from the {@code
+ * FederationApplicationHomeSubClusterStore} to a request to delete the mapping
+ * of home subcluster of a submitted application. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ */
+@Private
+@Unstable
+public abstract class DeleteApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static DeleteApplicationHomeSubClusterResponse newInstance() {
+ DeleteApplicationHomeSubClusterResponse response =
+ Records.newRecord(DeleteApplicationHomeSubClusterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..a64d22e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain the home sub-cluster for the specified
+ * {@link ApplicationId}.
+ */
+@Private
+@Unstable
+public abstract class GetApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static GetApplicationHomeSubClusterRequest newInstance(
+ ApplicationId appId) {
+ GetApplicationHomeSubClusterRequest appMapping =
+ Records.newRecord(GetApplicationHomeSubClusterRequest.class);
+ appMapping.setApplicationId(appId);
+ return appMapping;
+ }
+
+ /**
+ * Get the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @return the application identifier
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ /**
+ * Set the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @param applicationId the application identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..60735b3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The response sent by <code>Federation state
+ * store</code> to a query for the home subcluster of a newly submitted
+ * application.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class GetApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static GetApplicationHomeSubClusterResponse newInstance(
+ ApplicationHomeSubCluster applicationHomeSubCluster) {
+ GetApplicationHomeSubClusterResponse mapResponse =
+ Records.newRecord(GetApplicationHomeSubClusterResponse.class);
+ mapResponse.setApplicationHomeSubCluster(applicationHomeSubCluster);
+ return mapResponse;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @return the mapping of the application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationHomeSubCluster getApplicationHomeSubCluster();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @param applicationHomeSubCluster the mapping of the application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
new file mode 100644
index 0000000..6054972
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain the home sub-cluster mapping of all active
+ * applications.
+ */
+@Private
+@Unstable
+public abstract class GetApplicationsHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static GetApplicationsHomeSubClusterRequest newInstance() {
+ GetApplicationsHomeSubClusterRequest request =
+ Records.newRecord(GetApplicationsHomeSubClusterRequest.class);
+ return request;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java
new file mode 100644
index 0000000..ba3d2c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The response sent by <code>Federation state
+ * store</code> to a query for the home subcluster of all submitted
+ * applications.
+ *
+ * <p>
+ * The response includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class GetApplicationsHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static GetApplicationsHomeSubClusterResponse newInstance(
+ List<ApplicationHomeSubCluster> appsHomeSubClusters) {
+ GetApplicationsHomeSubClusterResponse mapResponse =
+ Records.newRecord(GetApplicationsHomeSubClusterResponse.class);
+ mapResponse.setAppsHomeSubClusters(appsHomeSubClusters);
+ return mapResponse;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
+ *
+ * @return the mapping of all submitted application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract List<ApplicationHomeSubCluster> getAppsHomeSubClusters();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
+ *
+ * @param appsHomeSubClusters the mapping of all submitted application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setAppsHomeSubClusters(
+ List<ApplicationHomeSubCluster> appsHomeSubClusters);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..eaa9252
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent by the <code>Router</code> to
+ * <code>Federation state store</code> to update the home subcluster of a newly
+ * submitted application.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class UpdateApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static UpdateApplicationHomeSubClusterRequest newInstance(
+ ApplicationHomeSubCluster applicationHomeSubCluster) {
+ UpdateApplicationHomeSubClusterRequest updateApplicationRequest =
+ Records.newRecord(UpdateApplicationHomeSubClusterRequest.class);
+ updateApplicationRequest
+ .setApplicationHomeSubCluster(applicationHomeSubCluster);
+ return updateApplicationRequest;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @return the mapping of the application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationHomeSubCluster getApplicationHomeSubCluster();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @param applicationHomeSubCluster the mapping of the application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..7434335
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * UpdateApplicationHomeSubClusterResponse contains the answer from the
+ * {@code FederationApplicationHomeSubClusterStore} to a request to register the
+ * home subcluster of a submitted application. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ */
+@Private
+@Unstable
+public abstract class UpdateApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static UpdateApplicationHomeSubClusterResponse newInstance() {
+ UpdateApplicationHomeSubClusterResponse response =
+ Records.newRecord(UpdateApplicationHomeSubClusterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..2387cde
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link AddApplicationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class AddApplicationHomeSubClusterRequestPBImpl
+ extends AddApplicationHomeSubClusterRequest {
+
+ private AddApplicationHomeSubClusterRequestProto proto =
+ AddApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private AddApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public AddApplicationHomeSubClusterRequestPBImpl() {
+ builder = AddApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public AddApplicationHomeSubClusterRequestPBImpl(
+ AddApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public AddApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = AddApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationHomeSubCluster getApplicationHomeSubCluster() {
+ AddApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasAppSubclusterMap()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAppSubclusterMap());
+ }
+
+ @Override
+ public void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationInfo) {
+ maybeInitBuilder();
+ if (applicationInfo == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ builder.setAppSubclusterMap(convertToProtoFormat(applicationInfo));
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..1415981
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link AddApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class AddApplicationHomeSubClusterResponsePBImpl
+ extends AddApplicationHomeSubClusterResponse {
+
+ private AddApplicationHomeSubClusterResponseProto proto =
+ AddApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private AddApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public AddApplicationHomeSubClusterResponsePBImpl() {
+ builder = AddApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public AddApplicationHomeSubClusterResponsePBImpl(
+ AddApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public AddApplicationHomeSubClusterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
new file mode 100644
index 0000000..7e6a564
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link ApplicationHomeSubCluster}.
+ */
+@Private
+@Unstable
+public class ApplicationHomeSubClusterPBImpl extends ApplicationHomeSubCluster {
+
+ private ApplicationHomeSubClusterProto proto =
+ ApplicationHomeSubClusterProto.getDefaultInstance();
+ private ApplicationHomeSubClusterProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ApplicationId applicationId = null;
+ private SubClusterId homeSubCluster = null;
+
+ public ApplicationHomeSubClusterPBImpl() {
+ builder = ApplicationHomeSubClusterProto.newBuilder();
+ }
+
+ public ApplicationHomeSubClusterPBImpl(ApplicationHomeSubClusterProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ApplicationHomeSubClusterProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationHomeSubClusterProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null) {
+ builder.setApplicationId(convertToProtoFormat(this.applicationId));
+ }
+ if (this.homeSubCluster != null) {
+ builder.setHomeSubCluster(convertToProtoFormat(this.homeSubCluster));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ ApplicationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ this.applicationId = convertFromProtoFormat(p.getApplicationId());
+ return applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ return;
+ }
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ public SubClusterId getHomeSubCluster() {
+ ApplicationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.homeSubCluster != null) {
+ return this.homeSubCluster;
+ }
+ if (!p.hasHomeSubCluster()) {
+ return null;
+ }
+ this.homeSubCluster = convertFromProtoFormat(p.getHomeSubCluster());
+ return this.homeSubCluster;
+ }
+
+ @Override
+ public void setHomeSubCluster(SubClusterId homeSubCluster) {
+ maybeInitBuilder();
+ if (homeSubCluster == null) {
+ builder.clearHomeSubCluster();
+ }
+ this.homeSubCluster = homeSubCluster;
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto subClusterId) {
+ return new SubClusterIdPBImpl(subClusterId);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId subClusterId) {
+ return ((SubClusterIdPBImpl) subClusterId).getProto();
+ }
+
+ private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
+ return new ApplicationIdPBImpl(appId);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId appId) {
+ return ((ApplicationIdPBImpl) appId).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..b4ef680
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link DeleteApplicationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class DeleteApplicationHomeSubClusterRequestPBImpl
+ extends DeleteApplicationHomeSubClusterRequest {
+
+ private DeleteApplicationHomeSubClusterRequestProto proto =
+ DeleteApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private DeleteApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public DeleteApplicationHomeSubClusterRequestPBImpl() {
+ builder = DeleteApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public DeleteApplicationHomeSubClusterRequestPBImpl(
+ DeleteApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public DeleteApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = DeleteApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ DeleteApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getApplicationId());
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ return;
+ }
+ builder.setApplicationId(convertToProtoFormat(applicationId));
+ }
+
+ private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
+ return new ApplicationIdPBImpl(appId);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId appId) {
+ return ((ApplicationIdPBImpl) appId).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..8a37b3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link DeleteApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class DeleteApplicationHomeSubClusterResponsePBImpl
+ extends DeleteApplicationHomeSubClusterResponse {
+
+ private DeleteApplicationHomeSubClusterResponseProto proto =
+ DeleteApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private DeleteApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public DeleteApplicationHomeSubClusterResponsePBImpl() {
+ builder = DeleteApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public DeleteApplicationHomeSubClusterResponsePBImpl(
+ DeleteApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public DeleteApplicationHomeSubClusterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..865d0c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class GetApplicationHomeSubClusterRequestPBImpl
+ extends GetApplicationHomeSubClusterRequest {
+
+ private GetApplicationHomeSubClusterRequestProto proto =
+ GetApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private GetApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ApplicationId applicationId = null;
+
+ public GetApplicationHomeSubClusterRequestPBImpl() {
+ builder = GetApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public GetApplicationHomeSubClusterRequestPBImpl(
+ GetApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null) {
+ builder.setApplicationId(convertToProtoFormat(this.applicationId));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ GetApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ this.applicationId = convertFromProtoFormat(p.getApplicationId());
+ return applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ return;
+ }
+ this.applicationId = applicationId;
+ }
+
+ private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
+ return new ApplicationIdPBImpl(appId);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId appId) {
+ return ((ApplicationIdPBImpl) appId).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..1180488
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetApplicationHomeSubClusterResponsePBImpl
+ extends GetApplicationHomeSubClusterResponse {
+
+ private GetApplicationHomeSubClusterResponseProto proto =
+ GetApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private GetApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetApplicationHomeSubClusterResponsePBImpl() {
+ builder = GetApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public GetApplicationHomeSubClusterResponsePBImpl(
+ GetApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationHomeSubClusterResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetApplicationHomeSubClusterResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationHomeSubCluster getApplicationHomeSubCluster() {
+ GetApplicationHomeSubClusterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasAppSubclusterMap()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAppSubclusterMap());
+ }
+
+ @Override
+ public void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationInfo) {
+ maybeInitBuilder();
+ if (applicationInfo == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ builder.setAppSubclusterMap(convertToProtoFormat(applicationInfo));
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..3ce8d74
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationsHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class GetApplicationsHomeSubClusterRequestPBImpl
+ extends GetApplicationsHomeSubClusterRequest {
+
+ private GetApplicationsHomeSubClusterRequestProto proto =
+ GetApplicationsHomeSubClusterRequestProto.getDefaultInstance();
+ private GetApplicationsHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetApplicationsHomeSubClusterRequestPBImpl() {
+ builder = GetApplicationsHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public GetApplicationsHomeSubClusterRequestPBImpl(
+ GetApplicationsHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationsHomeSubClusterRequestProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-13929. ADLS connector
should not check in contract-test-options.xml. (John Zhuge via lei)
Posted by su...@apache.org.
HADOOP-13929. ADLS connector should not check in contract-test-options.xml. (John Zhuge via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71c23c9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71c23c9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71c23c9f
Branch: refs/heads/YARN-2915
Commit: 71c23c9fc94cfdf58de80effbc3f51c0925d0cfe
Parents: 4ed33e9
Author: Lei Xu <le...@cloudera.com>
Authored: Mon Feb 13 13:33:13 2017 -0800
Committer: Lei Xu <le...@cloudera.com>
Committed: Mon Feb 13 13:33:13 2017 -0800
----------------------------------------------------------------------
.gitignore | 12 ++--
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 2 +-
.../src/site/markdown/index.md | 4 +-
.../fs/adl/live/AdlStorageConfiguration.java | 42 +++++++-------
.../src/test/resources/adls.xml | 11 ++++
.../test/resources/contract-test-options.xml | 61 --------------------
6 files changed, 39 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index eb98a3f..724162d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,10 @@ target
build
dependency-reduced-pom.xml
+# Filesystem contract test options and credentials
+auth-keys.xml
+azure-auth-keys.xml
+
# External tool builders
*/.externalToolBuilders
*/maven-eclipse.xml
@@ -24,8 +28,6 @@ dependency-reduced-pom.xml
hadoop-common-project/hadoop-kms/downloads/
hadoop-hdfs-project/hadoop-hdfs/downloads
hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
-hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/dist
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tmp
@@ -41,10 +43,4 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/testem.log
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/dist
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/tmp
yarnregistry.pdf
-hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
-hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
patchprocess/
-hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 3d41025..303b7bc 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -72,7 +72,7 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.*;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class AdlFileSystem extends FileSystem {
- static final String SCHEME = "adl";
+ public static final String SCHEME = "adl";
static final int DEFAULT_PORT = 443;
private URI uri;
private String userName;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index ced5cff..5037db6 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -224,7 +224,9 @@ commands demonstrate access to a storage account named `youraccount`.
## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
-A selection of tests can run against the Azure Data Lake Storage. To run tests against Adl storage. Please configure contract-test-options.xml with Adl account information mentioned in the above sections. Also turn on contract test execution flag to trigger tests against Azure Data Lake Storage.
+A selection of tests can run against the Azure Data Lake Storage. To run these
+tests, please create `src/test/resources/auth-keys.xml` with Adl account
+information mentioned in the above sections and the following properties.
<property>
<name>dfs.adl.test.contract.enable</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
index f7f0b71..7d6153d 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.fs.adl.live;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.apache.hadoop.util.ReflectionUtils;
import java.io.IOException;
import java.net.URI;
@@ -30,15 +32,19 @@ import java.net.URISyntaxException;
* Configure Adl storage file system.
*/
public final class AdlStorageConfiguration {
+ static final String CONTRACT_XML = "adls.xml";
+
private static final String CONTRACT_ENABLE_KEY =
"dfs.adl.test.contract.enable";
+ private static final boolean CONTRACT_ENABLE_DEFAULT = false;
+
+ private static final String FILE_SYSTEM_KEY =
+ String.format("test.fs.%s.name", AdlFileSystem.SCHEME);
- private static final String TEST_CONFIGURATION_FILE_NAME =
- "contract-test-options.xml";
- private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME =
- "adls.xml";
- private static final String KEY_FILE_SYSTEM_IMPL = "fs.contract.test.fs";
- private static final String KEY_FILE_SYSTEM = "test.fs.adl.name";
+ private static final String FILE_SYSTEM_IMPL_KEY =
+ String.format("fs.%s.impl", AdlFileSystem.SCHEME);
+ private static final Class<?> FILE_SYSTEM_IMPL_DEFAULT =
+ AdlFileSystem.class;
private static boolean isContractTestEnabled = false;
private static Configuration conf = null;
@@ -48,8 +54,7 @@ public final class AdlStorageConfiguration {
public synchronized static Configuration getConfiguration() {
Configuration newConf = new Configuration();
- newConf.addResource(TEST_CONFIGURATION_FILE_NAME);
- newConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME);
+ newConf.addResource(CONTRACT_XML);
return newConf;
}
@@ -58,7 +63,8 @@ public final class AdlStorageConfiguration {
conf = getConfiguration();
}
- isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false);
+ isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY,
+ CONTRACT_ENABLE_DEFAULT);
return isContractTestEnabled;
}
@@ -72,23 +78,15 @@ public final class AdlStorageConfiguration {
return null;
}
- String fileSystem = conf.get(KEY_FILE_SYSTEM);
+ String fileSystem = conf.get(FILE_SYSTEM_KEY);
if (fileSystem == null || fileSystem.trim().length() == 0) {
throw new IOException("Default file system not configured.");
}
- String fileSystemImpl = conf.get(KEY_FILE_SYSTEM_IMPL);
- if (fileSystemImpl == null || fileSystemImpl.trim().length() == 0) {
- throw new IOException(
- "Configuration " + KEY_FILE_SYSTEM_IMPL + "does not exist.");
- }
- FileSystem fs = null;
- try {
- fs = (FileSystem) Class.forName(fileSystemImpl).newInstance();
- } catch (Exception e) {
- throw new IOException("Could not instantiate the filesystem.");
- }
- fs.initialize(new URI(conf.get(KEY_FILE_SYSTEM)), conf);
+ Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
+ FILE_SYSTEM_IMPL_DEFAULT);
+ FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
+ fs.initialize(new URI(fileSystem), conf);
return fs;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
index 76a78b0..5bbdd6f 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml
@@ -12,6 +12,17 @@
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
+ <!--
+ To run live tests:
+ # Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
+ # Add property test.fs.adl.name to point to an ADLS filesystem URL
+ # Add property dfs.adl.test.contract.enable with value set to true
+ # Add the credentials for the service you are testing against
+ -->
+ <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
+ <fallback/>
+ </include>
+
<property>
<name>fs.contract.test.root-tests-enabled</name>
<value>true</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71c23c9f/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
deleted file mode 100644
index 646922e..0000000
--- a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<configuration>
- <property>
- <name>dfs.adls.oauth2.refresh.url</name>
- <value>
- </value>
- </property>
-
- <property>
- <name>dfs.adls.oauth2.credential</name>
- <value></value>
- </property>
-
- <property>
- <name>dfs.adls.oauth2.client.id</name>
- <value></value>
- </property>
-
- <property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
- <value>ClientCredential</value>
- <description>
- Supported provider type:
-
- "ClientCredential" : Client id and client credentials(Provided
- through configuration file) flow.
-
- "RefreshToken" : Client id and refresh token(Provided
- through configuration file)flow.
-
- "Custom" : Custom AAD token management.
- </description>
- </property>
-
- <property>
- <name>dfs.adl.test.contract.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>test.fs.adl.name</name>
- <value></value>
- </property>
-
- <property>
- <name>fs.contract.test.fs</name>
- <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
- </property>
-</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-11407. Document the missing
usages of OfflineImageViewer processors. Contributed by Yiqun Lin.
Posted by su...@apache.org.
HDFS-11407. Document the missing usages of OfflineImageViewer processors. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/719df99c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/719df99c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/719df99c
Branch: refs/heads/YARN-2915
Commit: 719df99c05a4e0bc62a7cad6d22c8dbe5d92bde0
Parents: 71c23c9
Author: Yiqun Lin <yq...@apache.org>
Authored: Tue Feb 14 18:48:08 2017 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Tue Feb 14 18:48:08 2017 +0800
----------------------------------------------------------------------
.../src/site/markdown/HdfsImageViewer.md | 63 ++++++++++++++++++++
1 file changed, 63 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/719df99c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index b677f6a..10d98b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -133,6 +133,69 @@ Applying the Offline Image Viewer with XML processor would result in the followi
</inode>
...remaining output omitted...
+### ReverseXML Processor
+
+ReverseXML processor is the opposite of the XML processor. Users can specify input XML file and output fsimage file via -i and -o command-line.
+
+ bash$ bin/hdfs oiv -p ReverseXML -i fsimage.xml -o fsimage
+
+This will reconstruct an fsimage from an XML file.
+
+### FileDistribution Processor
+
+FileDistribution processor can analyze file sizes in the namespace image. Users can specify maxSize (128GB by default) and step (2MB by default) in bytes via -maxSize and -step command-line.
+
+ bash$ bin/hdfs oiv -p FileDistribution -maxSize maxSize -step size -i fsimage -o output
+
+The processor will calculate how many files in the system fall into each segment. The output file is formatted as a tab separated two column table showed as the following output:
+
+ Size NumFiles
+ 4 1
+ 12 1
+ 16 1
+ 20 1
+ totalFiles = 4
+ totalDirectories = 2
+ totalBlocks = 4
+ totalSpace = 48
+ maxFileSize = 21
+
+To make the output result look more readable, users can specify -format option in addition.
+
+ bash$ bin/hdfs oiv -p FileDistribution -maxSize maxSize -step size -format -i fsimage -o output
+
+This would result in the following output:
+
+ Size Range NumFiles
+ (0 B, 4 B] 1
+ (8 B, 12 B] 1
+ (12 B, 16 B] 1
+ (16 B, 21 B] 1
+ totalFiles = 4
+ totalDirectories = 2
+ totalBlocks = 4
+ totalSpace = 48
+ maxFileSize = 21
+
+### Delimited Processor
+
+Delimited processor generates a text representation of the fsimage, with each element separated by a delimiter string (\t by default). Users can specify a new delimiter string by -delimiter option.
+
+ bash$ bin/hdfs oiv -p Delimited -delimiter delimiterString -i fsimage -o output
+
+In addition, users can specify a temporary dir to cache intermediate result by the following command:
+
+ bash$ bin/hdfs oiv -p Delimited -delimiter delimiterString -t temporaryDir -i fsimage -o output
+
+If not set, Delimited processor will construct the namespace in memory before outputting text. The output result of this processor should be like the following output:
+
+ Path Replication ModificationTime AccessTime PreferredBlockSize BlocksCount FileSize NSQUOTA DSQUOTA Permission UserName GroupName
+ / 0 2017-02-13 10:39 1970-01-01 08:00 0 0 0 9223372036854775807 -1 drwxr-xr-x root supergroup
+ /dir0 0 2017-02-13 10:39 1970-01-01 08:00 0 0 0 -1 -1 drwxr-xr-x root supergroup
+ /dir0/file0 1 2017-02-13 10:39 2017-02-13 10:39 134217728 1 1 0 0 -rw-r--r-- root supergroup
+ /dir0/file1 1 2017-02-13 10:39 2017-02-13 10:39 134217728 1 1 0 0 -rw-r--r-- root supergroup
+ /dir0/file2 1 2017-02-13 10:39 2017-02-13 10:39 134217728 1 1 0 0 -rw-r--r-- root supergroup
+
Options
-------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-5872. Add AlwayReject
policies for router and amrmproxy. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5872. Add AlwayReject policies for router and amrmproxy. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d9be842
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d9be842
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d9be842
Branch: refs/heads/YARN-2915
Commit: 8d9be8425ef44bb6bbdccd87a7d08fff4e455857
Parents: a34d8eb
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 22 18:37:30 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../amrmproxy/RejectAMRMProxyPolicy.java | 67 +++++++++++++++++
.../manager/RejectAllPolicyManager.java | 40 ++++++++++
.../policies/router/RejectRouterPolicy.java | 66 +++++++++++++++++
.../amrmproxy/TestRejectAMRMProxyPolicy.java | 78 ++++++++++++++++++++
.../manager/TestRejectAllPolicyManager.java | 40 ++++++++++
.../policies/router/TestRejectRouterPolicy.java | 63 ++++++++++++++++
6 files changed, 354 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d9be842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
new file mode 100644
index 0000000..3783df6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} that simply
+ * rejects all requests. Useful to prevent apps from accessing any sub-cluster.
+ */
+public class RejectAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ private Set<SubClusterId> knownClusterIds = new HashSet<>();
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+ // overrides initialize to avoid weight checks that do no apply for
+ // this policy.
+ FederationPolicyInitializationContextValidator.validate(policyContext,
+ this.getClass().getCanonicalName());
+ setPolicyContext(policyContext);
+ }
+
+ @Override
+ public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests) throws YarnException {
+ throw new FederationPolicyException("The policy configured for this queue "
+ + "rejects all routing requests by construction.");
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+ // This might be invoked for applications started with a previous policy,
+ // do nothing for this policy.
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d9be842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java
new file mode 100644
index 0000000..7bd7a1b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.RejectAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.RejectRouterPolicy;
+
+/**
+ * This class represents a simple implementation of a {@code
+ * FederationPolicyManager}.
+ *
+ * This policy rejects all reuqests for both router and amrmproxy routing. This
+ * is to be used to prevent applications in a specific queue (or if used as
+ * default for non-configured queues) from accessing cluster resources.
+ */
+public class RejectAllPolicyManager extends AbstractPolicyManager {
+
+ public RejectAllPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = RejectRouterPolicy.class;
+ amrmProxyFederationPolicy = RejectAMRMProxyPolicy.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d9be842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
new file mode 100644
index 0000000..faf3279
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * This {@link FederationRouterPolicy} simply rejects all incoming requests.
+ * This is useful to prevent applications running in a queue to be run
+ * anywhere in the federated cluster.
+ */
+public class RejectRouterPolicy extends AbstractRouterPolicy {
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+ setPolicyContext(federationPolicyContext);
+ }
+
+ /**
+ * The policy always reject requests.
+ *
+ * @param appSubmissionContext the context for the app being submitted.
+ *
+ * @return (never).
+ *
+ * @throws YarnException (always) to prevent applications in this queue to
+ * be run anywhere in the federated cluster.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+
+ // run standard validation, as error might differ
+ validate(appSubmissionContext);
+
+ throw new FederationPolicyException("The policy configured for this queue"
+ + " (" + appSubmissionContext.getQueue() + ") reject all routing "
+ + "requests by construction. Application " + appSubmissionContext
+ .getApplicationId() + " cannot be routed to any RM.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d9be842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
new file mode 100644
index 0000000..41e7fed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link RejectAMRMProxyPolicy}.
+ */
+public class TestRejectAMRMProxyPolicy
+ extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new RejectAMRMProxyPolicy());
+ // needed for base test to work
+ setPolicyInfo(mock(WeightedPolicyInfo.class));
+
+ for (int i = 1; i <= 2; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ mock(WeightedPolicyInfo.class), getActiveSubclusters());
+
+ }
+
+ @Test (expected = FederationPolicyException.class)
+ public void testSplitAllocateRequest() throws Exception {
+ // verify the request is broadcasted to all subclusters
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+ }
+
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d9be842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java
new file mode 100644
index 0000000..e4dc7f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.RejectAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.RejectRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link RejectAllPolicyManager}.
+ */
+public class TestRejectAllPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ // config policy
+ wfp = new RejectAllPolicyManager();
+ wfp.setQueue("queue1");
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = RejectAllPolicyManager.class;
+ expectedAMRMProxyPolicy = RejectAMRMProxyPolicy.class;
+ expectedRouterPolicy = RejectRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d9be842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
new file mode 100644
index 0000000..049ebbf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link RejectRouterPolicy}. Tests that one of the
+ * active subcluster is chosen.
+ */
+public class TestRejectRouterPolicy extends BaseRouterPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new RejectRouterPolicy());
+
+ // setting up the active sub-clusters for this test
+ setMockActiveSubclusters(2);
+
+ // initialize policy with context
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+
+ }
+
+ @Test(expected = FederationPolicyException.class)
+ public void testNoClusterIsChosen() throws YarnException {
+ ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ }
+
+ @Override
+ @Test(expected = FederationPolicyException.class)
+ public void testNullQueueRouting() throws YarnException {
+ FederationRouterPolicy localPolicy = (FederationRouterPolicy) getPolicy();
+ ApplicationSubmissionContext applicationSubmissionContext =
+ ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+ false, false, 0, Resources.none(), null, false, null, null);
+ localPolicy.getHomeSubcluster(applicationSubmissionContext);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-5323. Policies APIs for
Federation Router and AMRMProxy policies. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5323. Policies APIs for Federation Router and AMRMProxy policies. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/288c55e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/288c55e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/288c55e7
Branch: refs/heads/YARN-2915
Commit: 288c55e7cf0a9245bf6259b31ebd28126b12c29d
Parents: ed985a3
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Sep 7 17:33:34 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../policies/ConfigurableFederationPolicy.java | 44 +++++++
.../policies/FederationPolicyConfigurator.java | 91 +++++++++++++
.../FederationPolicyInitializationContext.java | 109 ++++++++++++++++
...ionPolicyInitializationContextValidator.java | 82 ++++++++++++
.../policies/FederationPolicyWriter.java | 45 +++++++
.../amrmproxy/FederationAMRMProxyPolicy.java | 66 ++++++++++
.../policies/amrmproxy/package-info.java | 20 +++
.../exceptions/FederationPolicyException.java | 33 +++++
...FederationPolicyInitializationException.java | 33 +++++
.../NoActiveSubclustersException.java | 27 ++++
.../exceptions/UnknownSubclusterException.java | 28 ++++
.../policies/exceptions/package-info.java | 20 +++
.../federation/policies/package-info.java | 20 +++
.../policies/router/FederationRouterPolicy.java | 45 +++++++
.../policies/router/package-info.java | 20 +++
...ionPolicyInitializationContextValidator.java | 128 +++++++++++++++++++
.../utils/FederationPoliciesTestUtil.java | 83 ++++++++++++
17 files changed, 894 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
new file mode 100644
index 0000000..fd6ceea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+/**
+ * This interface provides a general method to reinitialize a policy. The
+ * semantics are try-n-swap, so in case of an exception is thrown the
+ * implmentation must ensure the previous state and configuration is preserved.
+ */
+public interface ConfigurableFederationPolicy {
+
+ /**
+ * This method is invoked to initialize of update the configuration of
+ * policies. The implementor should provide try-n-swap semantics, and retain
+ * state if possible.
+ *
+ * @param federationPolicyInitializationContext the new context to provide to
+ * implementor.
+ *
+ * @throws FederationPolicyInitializationException in case the initialization
+ * fails.
+ */
+ void reinitialize(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext)
+ throws FederationPolicyInitializationException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
new file mode 100644
index 0000000..fdc3857
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+import org.apache.hadoop.yarn.server.federation.policies.router
+ .FederationRouterPolicy;
+
+/**
+ * Implementors of this interface are capable to instantiate and (re)initalize
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} based on
+ * a {@link FederationPolicyInitializationContext}. The reason to bind these two
+ * policies together is to make sure we remain consistent across the router and
+ * amrmproxy policy decisions.
+ */
+public interface FederationPolicyConfigurator {
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
+ * current context, otherwise a new instance initialized with the current
+ * context is provided. If the instance is compatible with the current class
+ * the implementors should attempt to reinitalize (retaining state). To affect
+ * a complete policy reset oldInstance should be null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationAMRMProxyPolicy
+ }.
+ *
+ * @throws FederationPolicyInitializationException if the initialization
+ * cannot be completed
+ * properly. The oldInstance
+ * should be still valid in
+ * case of failed
+ * initialization.
+ */
+ FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationRouterPolicy} reinitialized with the current
+ * context, otherwise a new instance initialized with the current context is
+ * provided. If the instance is compatible with the current class the
+ * implementors should attempt to reinitalize (retaining state). To affect a
+ * complete policy reset oldInstance shoulb be set to null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationRouterPolicy}.
+ *
+ * @throws FederationPolicyInitializationException if the initalization cannot
+ * be completed properly. The
+ * oldInstance should be still
+ * valid in case of failed
+ * initialization.
+ */
+ FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
new file mode 100644
index 0000000..879ccee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+
+/**
+ * Context to (re)initialize a {@code FederationAMRMProxyPolicy} and {@code
+ * FederationRouterPolicy}.
+ */
+public class FederationPolicyInitializationContext {
+
+ private SubClusterPolicyConfiguration federationPolicyConfiguration;
+ private SubClusterResolver federationSubclusterResolver;
+ private FederationStateStoreFacade federationStateStoreFacade;
+
+ public FederationPolicyInitializationContext() {
+ federationPolicyConfiguration = null;
+ federationSubclusterResolver = null;
+ federationStateStoreFacade = null;
+ }
+
+ public FederationPolicyInitializationContext(SubClusterPolicyConfiguration
+ policy, SubClusterResolver resolver, FederationStateStoreFacade
+ storeFacade) {
+ this.federationPolicyConfiguration = policy;
+ this.federationSubclusterResolver = resolver;
+ this.federationStateStoreFacade = storeFacade;
+ }
+
+
+ /**
+ * Getter for the {@link SubClusterPolicyConfiguration}.
+ *
+ * @return the {@link SubClusterPolicyConfiguration} to be used for
+ * initialization.
+ */
+ public SubClusterPolicyConfiguration getSubClusterPolicyConfiguration() {
+ return federationPolicyConfiguration;
+ }
+
+ /**
+ * Setter for the {@link SubClusterPolicyConfiguration}.
+ *
+ * @param federationPolicyConfiguration the
+ * {@link SubClusterPolicyConfiguration}
+ * to be used for initialization.
+ */
+ public void setFederationPolicyConfiguration(
+ SubClusterPolicyConfiguration federationPolicyConfiguration) {
+ this.federationPolicyConfiguration = federationPolicyConfiguration;
+ }
+
+ /**
+ * Getter for the {@link SubClusterResolver}.
+ *
+ * @return the {@link SubClusterResolver} to be used for initialization.
+ */
+ public SubClusterResolver getFederationSubclusterResolver() {
+ return federationSubclusterResolver;
+ }
+
+ /**
+ * Setter for the {@link SubClusterResolver}.
+ *
+ * @param federationSubclusterResolver the {@link SubClusterResolver} to be
+ * used for initialization.
+ */
+ public void setFederationSubclusterResolver(
+ SubClusterResolver federationSubclusterResolver) {
+ this.federationSubclusterResolver = federationSubclusterResolver;
+ }
+
+ /**
+ * Getter for the {@link FederationStateStoreFacade}.
+ *
+ * @return the facade.
+ */
+ public FederationStateStoreFacade getFederationStateStoreFacade() {
+ return federationStateStoreFacade;
+ }
+
+ /**
+ * Setter for the {@link FederationStateStoreFacade}.
+ *
+ * @param federationStateStoreFacade the facade.
+ */
+ public void setFederationStateStoreFacade(
+ FederationStateStoreFacade federationStateStoreFacade) {
+ this.federationStateStoreFacade = federationStateStoreFacade;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
new file mode 100644
index 0000000..31f83d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+/**
+ * Helper class used to factor out common validation steps for policies.
+ */
+public final class FederationPolicyInitializationContextValidator {
+
+ private FederationPolicyInitializationContextValidator() {
+ //disable constructor per checkstyle
+ }
+
+ public static void validate(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ String myType) throws FederationPolicyInitializationException {
+
+ if (myType == null) {
+ throw new FederationPolicyInitializationException("The myType parameter"
+ + " should not be null.");
+ }
+
+ if (federationPolicyInitializationContext == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationPolicyInitializationContext provided is null. Cannot"
+ + " reinitalize "
+ + "successfully.");
+ }
+
+ if (federationPolicyInitializationContext.getFederationStateStoreFacade()
+ == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationStateStoreFacade provided is null. Cannot"
+ + " reinitalize successfully.");
+ }
+
+ if (federationPolicyInitializationContext.getFederationSubclusterResolver()
+ == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationStateStoreFacase provided is null. Cannot"
+ + " reinitalize successfully.");
+ }
+
+ if (federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
+ == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationSubclusterResolver provided is null. Cannot "
+ + "reinitalize successfully.");
+ }
+
+ String intendedType =
+ federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
+ .getType();
+
+ if (!myType.equals(intendedType)) {
+ throw new FederationPolicyInitializationException(
+ "The FederationPolicyConfiguration carries a type (" + intendedType
+ + ") different then mine (" + myType
+ + "). Cannot reinitialize successfully.");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
new file mode 100644
index 0000000..5034b7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+/**
+ * Implementors of this class are able to serializeConf the configuraiton of a
+ * policy as a {@link SubClusterPolicyConfiguration}. This is used during the
+ * lifetime of a policy from the admin APIs or policy engine to serializeConf
+ * the policy into the policy store.
+ */
+public interface FederationPolicyWriter {
+
+ /**
+ /**
+ * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
+ * This is to be used when writing a policy object in the federation policy
+ * store.
+ *
+ * @return a valid policy configuration representing this object
+ * parametrization.
+ *
+ * @throws FederationPolicyInitializationException if the current state cannot
+ * be serialized properly
+ */
+ SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
new file mode 100644
index 0000000..4a3305c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Implementors of this interface provide logic to split the list of {@link
+ * ResourceRequest}s received by the AM among various RMs.
+ */
+public interface FederationAMRMProxyPolicy
+ extends ConfigurableFederationPolicy {
+
+ /**
+ * Splits the {@link ResourceRequest}s from the client across one or more
+ * sub-clusters based on the policy semantics (e.g., broadcast, load-based).
+ *
+ * @param resourceRequests the list of {@link ResourceRequest}s from the
+ * AM to be split
+ *
+ * @return map of sub-cluster as identified by {@link SubClusterId} to the
+ * list of {@link ResourceRequest}s that should be forwarded to it
+ *
+ * @throws YarnException in case the request is malformed or no viable
+ * sub-clusters can be found.
+ */
+ Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests)
+ throws YarnException;
+
+ /**
+ * This method should be invoked to notify the policy about responses being
+ * received. This is useful for stateful policies that make decisions based on
+ * previous responses being received.
+ *
+ * @param subClusterId the id of the subcluster sending the notification
+ * @param response the response received from one of the RMs
+ *
+ * @throws YarnException in case the response is not valid
+ */
+ void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
new file mode 100644
index 0000000..99da20b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** AMRMPRoxy policies. **/
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java
new file mode 100644
index 0000000..24fe421
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Generic policy exception.
+ */
+public class FederationPolicyException extends YarnException {
+ public FederationPolicyException(String s) {
+ super(s);
+ }
+
+ public FederationPolicyException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java
new file mode 100644
index 0000000..fcc09c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+/**
+ * This exception is thrown when the initialization of a federation policy is
+ * not successful.
+ */
+public class FederationPolicyInitializationException
+ extends FederationPolicyException {
+ public FederationPolicyInitializationException(String message) {
+ super(message);
+ }
+
+ public FederationPolicyInitializationException(Throwable j) {
+ super(j);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java
new file mode 100644
index 0000000..a427944
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+/**
+ * This exception is thrown when policies cannot locate any active cluster.
+ */
+public class NoActiveSubclustersException extends FederationPolicyException {
+ public NoActiveSubclustersException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java
new file mode 100644
index 0000000..8a0fb4f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+/**
+ * This exception is thrown whenever a policy is given a {@code SubClusterId}
+ * that is unknown.
+ */
+public class UnknownSubclusterException extends FederationPolicyException {
+ public UnknownSubclusterException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
new file mode 100644
index 0000000..3318da9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Exceptions for policies. **/
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
new file mode 100644
index 0000000..7d9a121
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Federation Policies. **/
+package org.apache.hadoop.yarn.server.federation.policies;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
new file mode 100644
index 0000000..42c86cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * Implements the logic for determining the routing of an application submission
+ * based on a policy.
+ */
+public interface FederationRouterPolicy extends ConfigurableFederationPolicy {
+
+ /**
+ * Determines the sub-cluster that the user application submision should be
+ * routed to.
+ *
+ * @param appSubmissionContext the context for the app being submitted.
+ *
+ * @return the sub-cluster as identified by {@link SubClusterId} to route the
+ * request to.
+ *
+ * @throws YarnException if the policy cannot determine a viable subcluster.
+ */
+ SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
new file mode 100644
index 0000000..5d0fcb6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Router policies. **/
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
new file mode 100644
index 0000000..4ec04d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Test class for {@link FederationPolicyInitializationContextValidator}.
+ */
+public class TestFederationPolicyInitializationContextValidator {
+
+ private SubClusterPolicyConfiguration goodConfig;
+ private SubClusterResolver goodSR;
+ private FederationStateStoreFacade goodFacade;
+ private FederationPolicyInitializationContext context;
+
+ @Before
+ public void setUp() throws Exception {
+ goodFacade = FederationPoliciesTestUtil.initFacade();
+ goodConfig =
+ new MockPolicyManager().serializeConf();
+ goodSR =FederationPoliciesTestUtil.initResolver();
+ context = new
+ FederationPolicyInitializationContext(goodConfig, goodSR, goodFacade);
+ }
+
+ @Test
+ public void correcInit() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullContext() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(null,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullType() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(context, null);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void wrongType() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(context,
+ "WrongType");
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullConf() throws Exception {
+ context.setFederationPolicyConfiguration(null);
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullResolver() throws Exception {
+ context.setFederationSubclusterResolver(null);
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullFacade() throws Exception {
+ context.setFederationStateStoreFacade(null);
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ private class MockPolicyManager
+ implements FederationPolicyWriter, FederationPolicyConfigurator {
+ @Override
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+ return null;
+ }
+
+ @Override
+ public FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+ return null;
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = ByteBuffer.allocate(0);
+ return SubClusterPolicyConfiguration
+ .newInstance("queue1", this.getClass().getCanonicalName(), buf);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/288c55e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
new file mode 100644
index 0000000..8c2115b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+
+import java.net.URL;
+import java.util.ArrayList;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Support class providing common initialization methods to test federation
+ * policies.
+ */
+public final class FederationPoliciesTestUtil {
+
+ private FederationPoliciesTestUtil() {
+ // disabled.
+ }
+
+ /**
+ * Initialize a {@link SubClusterResolver}.
+ *
+ * @return a subcluster resolver for tests.
+ */
+ public static SubClusterResolver initResolver() {
+ YarnConfiguration conf = new YarnConfiguration();
+ SubClusterResolver resolver =
+ new DefaultSubClusterResolverImpl();
+ URL url =
+ Thread.currentThread().getContextClassLoader().getResource("nodes");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'nodes' dummy file in classpath");
+ }
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, url.getPath());
+ resolver.setConf(conf);
+ resolver.load();
+ return resolver;
+ }
+
+ /**
+ * Initialiaze a main-memory {@link FederationStateStoreFacade} used for
+ * testing, wiht a mock resolver.
+ *
+ * @return the facade.
+ *
+ * @throws YarnException in case the initialization is not successful.
+ */
+ public static FederationStateStoreFacade initFacade() throws YarnException {
+ FederationStateStoreFacade goodFacade = FederationStateStoreFacade
+ .getInstance();
+ FederationStateStore fss = mock(FederationStateStore.class);
+ GetSubClustersInfoResponse response = GetSubClustersInfoResponse
+ .newInstance(new ArrayList<>());
+ when(fss.getSubClusters(any())).thenReturn(response);
+ goodFacade.reinitialize(fss, new Configuration());
+ return goodFacade;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-5905. Update the RM webapp
host that is reported as part of Federation membership to current primary
RM's IP.
Posted by su...@apache.org.
YARN-5905. Update the RM webapp host that is reported as part of Federation membership to current primary RM's IP.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a34d8ebf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a34d8ebf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a34d8ebf
Branch: refs/heads/YARN-2915
Commit: a34d8ebf73be2bb9f9a55f674027510fe40da691
Parents: fc338ed
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 22 18:30:40 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../federation/FederationStateStoreService.java | 4 ++--
.../federation/TestFederationRMStateStoreService.java | 11 ++++++++++-
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a34d8ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
index 9a01d7e..530184f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
@@ -177,8 +177,8 @@ public class FederationStateStoreService extends AbstractService
config.getSocketAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
- String webAppAddress =
- WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(config);
+ String webAppAddress = getServiceAddress(NetUtils
+ .createSocketAddr(WebAppUtils.getRMWebAppURLWithScheme(config)));
SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
amRMAddress, clientRMAddress, rmAdminAddress, webAppAddress,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a34d8ebf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
index 30f69b5..d92a793 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.federation;
import java.io.IOException;
import java.io.StringReader;
+import java.net.UnknownHostException;
import javax.xml.bind.JAXBException;
@@ -157,12 +158,20 @@ public class TestFederationRMStateStoreService {
}
private String checkSubClusterInfo(SubClusterState state)
- throws YarnException {
+ throws YarnException, UnknownHostException {
Assert.assertNotNull(stateStore.getSubCluster(request));
SubClusterInfo response =
stateStore.getSubCluster(request).getSubClusterInfo();
Assert.assertEquals(state, response.getState());
Assert.assertTrue(response.getLastHeartBeat() >= lastHearbeatTS);
+ String expectedAddress =
+ (response.getClientRMServiceAddress().split(":"))[0];
+ Assert.assertEquals(expectedAddress,
+ (response.getAMRMServiceAddress().split(":"))[0]);
+ Assert.assertEquals(expectedAddress,
+ (response.getRMAdminServiceAddress().split(":"))[0]);
+ Assert.assertEquals(expectedAddress,
+ (response.getRMWebServiceAddress().split(":"))[0]);
lastHearbeatTS = response.getLastHeartBeat();
return response.getCapability();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-5612. Return SubClusterId in
FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover.
(Giovanni Matteo Fumarola via Subru).
Posted by su...@apache.org.
YARN-5612. Return SubClusterId in FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover. (Giovanni Matteo Fumarola via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2994dff4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2994dff4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2994dff4
Branch: refs/heads/YARN-2915
Commit: 2994dff465922d30ea88f08a7821eff047967707
Parents: 5ef27f1
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Sep 1 13:55:54 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../utils/FederationStateStoreFacade.java | 11 ++++---
.../utils/TestFederationStateStoreFacade.java | 30 ++++++++++++++++++++
2 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2994dff4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index f1c8218..66a0b60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
@@ -298,13 +299,15 @@ public final class FederationStateStoreFacade {
*
* @param appHomeSubCluster the mapping of the application to it's home
* sub-cluster
+ * @return the stored Subcluster from StateStore
* @throws YarnException if the call to the state store is unsuccessful
*/
- public void addApplicationHomeSubCluster(
+ public SubClusterId addApplicationHomeSubCluster(
ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
- stateStore.addApplicationHomeSubCluster(
- AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
- return;
+ AddApplicationHomeSubClusterResponse response =
+ stateStore.addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+ return response.getHomeSubCluster();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2994dff4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
index 53f4f84..d46bef0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
@@ -145,4 +146,33 @@ public class TestFederationStateStoreFacade {
}
}
+ @Test
+ public void testAddApplicationHomeSubCluster() throws YarnException {
+
+ // Inserting <AppId, Home1> into FederationStateStore
+ ApplicationId appId = ApplicationId.newInstance(clusterTs, numApps + 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("Home1");
+
+ ApplicationHomeSubCluster appHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId1);
+
+ SubClusterId result =
+ facade.addApplicationHomeSubCluster(appHomeSubCluster);
+
+ Assert.assertEquals(facade.getApplicationHomeSubCluster(appId), result);
+ Assert.assertEquals(subClusterId1, result);
+
+ // Inserting <AppId, Home2> into FederationStateStore.
+ // The application is already present.
+ // FederationFacade will return Home1 as SubClusterId.
+ SubClusterId subClusterId2 = SubClusterId.newInstance("Home2");
+ appHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
+
+ result = facade.addApplicationHomeSubCluster(appHomeSubCluster);
+
+ Assert.assertEquals(facade.getApplicationHomeSubCluster(appId), result);
+ Assert.assertEquals(subClusterId1, result);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-11026. Convert
BlockTokenIdentifier to use Protobuf. Contributed by Ewan Higgs.
Posted by su...@apache.org.
HDFS-11026. Convert BlockTokenIdentifier to use Protobuf. Contributed by Ewan Higgs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ed33e9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ed33e9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ed33e9c
Branch: refs/heads/YARN-2915
Commit: 4ed33e9ca3d85568e3904753a3ef61a85f801838
Parents: 646c6d6
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Feb 13 11:27:48 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Mon Feb 13 11:29:18 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 51 ++++
.../token/block/BlockTokenIdentifier.java | 89 +++++-
.../src/main/proto/hdfs.proto | 33 +++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +
.../token/block/BlockTokenSecretManager.java | 18 +-
.../hadoop/hdfs/server/balancer/KeyManager.java | 6 +-
.../server/blockmanagement/BlockManager.java | 9 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 5 +-
.../src/main/resources/hdfs-default.xml | 9 +
.../security/token/block/TestBlockToken.java | 297 +++++++++++++++++--
10 files changed, 480 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 0180828..ad80bc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -121,9 +121,11 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmI
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenSecretProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CryptoProtocolVersionProto;
@@ -584,6 +586,55 @@ public class PBHelperClient {
return blockTokens;
}
+ public static AccessModeProto convert(BlockTokenIdentifier.AccessMode aMode) {
+ switch (aMode) {
+ case READ: return AccessModeProto.READ;
+ case WRITE: return AccessModeProto.WRITE;
+ case COPY: return AccessModeProto.COPY;
+ case REPLACE: return AccessModeProto.REPLACE;
+ default:
+ throw new IllegalArgumentException("Unexpected AccessMode: " + aMode);
+ }
+ }
+
+ public static BlockTokenIdentifier.AccessMode convert(
+ AccessModeProto accessModeProto) {
+ switch (accessModeProto) {
+ case READ: return BlockTokenIdentifier.AccessMode.READ;
+ case WRITE: return BlockTokenIdentifier.AccessMode.WRITE;
+ case COPY: return BlockTokenIdentifier.AccessMode.COPY;
+ case REPLACE: return BlockTokenIdentifier.AccessMode.REPLACE;
+ default:
+ throw new IllegalArgumentException("Unexpected AccessModeProto: " +
+ accessModeProto);
+ }
+ }
+
+ public static BlockTokenSecretProto convert(
+ BlockTokenIdentifier blockTokenSecret) {
+ BlockTokenSecretProto.Builder builder =
+ BlockTokenSecretProto.newBuilder();
+ builder.setExpiryDate(blockTokenSecret.getExpiryDate());
+ builder.setKeyId(blockTokenSecret.getKeyId());
+ String userId = blockTokenSecret.getUserId();
+ if (userId != null) {
+ builder.setUserId(userId);
+ }
+
+ String blockPoolId = blockTokenSecret.getBlockPoolId();
+ if (blockPoolId != null) {
+ builder.setBlockPoolId(blockPoolId);
+ }
+
+ builder.setBlockId(blockTokenSecret.getBlockId());
+
+ for (BlockTokenIdentifier.AccessMode aMode :
+ blockTokenSecret.getAccessModes()) {
+ builder.addModes(convert(aMode));
+ }
+ return builder.build();
+ }
+
static public DatanodeInfo convert(DatanodeInfoProto di) {
if (di == null) {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
index 3f2c9ca..28e7acc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
@@ -19,11 +19,16 @@
package org.apache.hadoop.hdfs.security.token.block;
import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.util.EnumSet;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -44,20 +49,22 @@ public class BlockTokenIdentifier extends TokenIdentifier {
private String blockPoolId;
private long blockId;
private final EnumSet<AccessMode> modes;
+ private boolean useProto;
private byte [] cache;
public BlockTokenIdentifier() {
- this(null, null, 0, EnumSet.noneOf(AccessMode.class));
+ this(null, null, 0, EnumSet.noneOf(AccessMode.class), false);
}
public BlockTokenIdentifier(String userId, String bpid, long blockId,
- EnumSet<AccessMode> modes) {
+ EnumSet<AccessMode> modes, boolean useProto) {
this.cache = null;
this.userId = userId;
this.blockPoolId = bpid;
this.blockId = blockId;
this.modes = modes == null ? EnumSet.noneOf(AccessMode.class) : modes;
+ this.useProto = useProto;
}
@Override
@@ -144,9 +151,45 @@ public class BlockTokenIdentifier extends TokenIdentifier {
^ (blockPoolId == null ? 0 : blockPoolId.hashCode());
}
+ /**
+ * readFields peeks at the first byte of the DataInput and determines if it
+ * was written using WritableUtils ("Legacy") or Protobuf. We can do this
+ * because we know the first field is the Expiry date.
+ *
+ * In the case of the legacy buffer, the expiry date is a VInt, so the size
+ * (which should always be >1) is encoded in the first byte - which is
+ * always negative due to this encoding. However, there are sometimes null
+ * BlockTokenIdentifier written so we also need to handle the case there
+ * the first byte is also 0.
+ *
+ * In the case of protobuf, the first byte is a type tag for the expiry date
+ * which is written as <code>(field_number << 3 | wire_type</code>.
+ * So as long as the field_number is less than 16, but also positive, then
+ * we know we have a Protobuf.
+ *
+ * @param in <code>DataInput</code> to deserialize this object from.
+ * @throws IOException
+ */
@Override
public void readFields(DataInput in) throws IOException {
this.cache = null;
+
+ final DataInputStream dis = (DataInputStream)in;
+ if (!dis.markSupported()) {
+ throw new IOException("Could not peek first byte.");
+ }
+ dis.mark(1);
+ final byte firstByte = dis.readByte();
+ dis.reset();
+ if (firstByte <= 0) {
+ readFieldsLegacy(dis);
+ } else {
+ readFieldsProtobuf(dis);
+ }
+ }
+
+ @VisibleForTesting
+ void readFieldsLegacy(DataInput in) throws IOException {
expiryDate = WritableUtils.readVLong(in);
keyId = WritableUtils.readVInt(in);
userId = WritableUtils.readString(in);
@@ -157,10 +200,44 @@ public class BlockTokenIdentifier extends TokenIdentifier {
for (int i = 0; i < length; i++) {
modes.add(WritableUtils.readEnum(in, AccessMode.class));
}
+ useProto = false;
+ }
+
+ @VisibleForTesting
+ void readFieldsProtobuf(DataInput in) throws IOException {
+ BlockTokenSecretProto blockTokenSecretProto =
+ BlockTokenSecretProto.parseFrom((DataInputStream)in);
+ expiryDate = blockTokenSecretProto.getExpiryDate();
+ keyId = blockTokenSecretProto.getKeyId();
+ if (blockTokenSecretProto.hasUserId()) {
+ userId = blockTokenSecretProto.getUserId();
+ } else {
+ userId = null;
+ }
+ if (blockTokenSecretProto.hasBlockPoolId()) {
+ blockPoolId = blockTokenSecretProto.getBlockPoolId();
+ } else {
+ blockPoolId = null;
+ }
+ blockId = blockTokenSecretProto.getBlockId();
+ for (int i = 0; i < blockTokenSecretProto.getModesCount(); i++) {
+ AccessModeProto accessModeProto = blockTokenSecretProto.getModes(i);
+ modes.add(PBHelperClient.convert(accessModeProto));
+ }
+ useProto = true;
}
@Override
public void write(DataOutput out) throws IOException {
+ if (useProto) {
+ writeProtobuf(out);
+ } else {
+ writeLegacy(out);
+ }
+ }
+
+ @VisibleForTesting
+ void writeLegacy(DataOutput out) throws IOException {
WritableUtils.writeVLong(out, expiryDate);
WritableUtils.writeVInt(out, keyId);
WritableUtils.writeString(out, userId);
@@ -172,6 +249,12 @@ public class BlockTokenIdentifier extends TokenIdentifier {
}
}
+ @VisibleForTesting
+ void writeProtobuf(DataOutput out) throws IOException {
+ BlockTokenSecretProto secret = PBHelperClient.convert(this);
+ out.write(secret.toByteArray());
+ }
+
@Override
public byte[] getBytes() {
if(cache == null) cache = super.getBytes();
@@ -186,4 +269,4 @@ public class BlockTokenIdentifier extends TokenIdentifier {
return KIND_NAME;
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 1414120..8a039d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -514,3 +514,36 @@ message RollingUpgradeStatusProto {
message StorageUuidsProto {
repeated string storageUuids = 1;
}
+
+/**
+ * File access permissions mode.
+ */
+enum AccessModeProto {
+ READ = 1;
+ WRITE = 2;
+ COPY = 3;
+ REPLACE = 4;
+}
+
+/**
+ * Secret information for the BlockKeyProto. This is not sent on the wire as
+ * such but is used to pack a byte array and encrypted and put in
+ * BlockKeyProto.bytes
+ * When adding further fields, make sure they are optional as they would
+ * otherwise not be backwards compatible.
+ *
+ * Note: As part of the migration from WritableUtils based tokens (aka "legacy")
+ * to Protocol Buffers, we use the first byte to determine the type. If the
+ * first byte is <=0 then it is a legacy token. This means that when using
+ * protobuf tokens, the the first field sent must have a `field_number` less
+ * than 16 to make sure that the first byte is positive. Otherwise it could be
+ * parsed as a legacy token. See HDFS-11026 for more discussion.
+ */
+message BlockTokenSecretProto {
+ optional uint64 expiryDate = 1;
+ optional uint32 keyId = 2;
+ optional string userId = 3;
+ optional string blockPoolId = 4;
+ optional uint64 blockId = 5;
+ repeated AccessModeProto modes = 6;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 10a521b..cf1d21a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -641,6 +641,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_DEFAULT = 600L;
public static final String DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY = "dfs.block.access.token.lifetime";
public static final long DFS_BLOCK_ACCESS_TOKEN_LIFETIME_DEFAULT = 600L;
+ public static final String DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE = "dfs.block.access.token.protobuf.enable";
+ public static final boolean DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE_DEFAULT = false;
public static final String DFS_BLOCK_REPLICATOR_CLASSNAME_KEY = "dfs.block.replicator.classname";
public static final Class<BlockPlacementPolicyDefault> DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT = BlockPlacementPolicyDefault.class;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
index ba08740..a3100d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
@@ -75,6 +75,7 @@ public class BlockTokenSecretManager extends
private final int intRange;
private final int nnRangeStart;
+ private final boolean useProto;
private final SecureRandom nonceGenerator = new SecureRandom();
@@ -83,11 +84,13 @@ public class BlockTokenSecretManager extends
*
* @param keyUpdateInterval how often a new key will be generated
* @param tokenLifetime how long an individual token is valid
+ * @param useProto should we use new protobuf style tokens
*/
public BlockTokenSecretManager(long keyUpdateInterval,
- long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
+ long tokenLifetime, String blockPoolId, String encryptionAlgorithm,
+ boolean useProto) {
this(false, keyUpdateInterval, tokenLifetime, blockPoolId,
- encryptionAlgorithm, 0, 1);
+ encryptionAlgorithm, 0, 1, useProto);
}
/**
@@ -102,8 +105,9 @@ public class BlockTokenSecretManager extends
*/
public BlockTokenSecretManager(long keyUpdateInterval,
long tokenLifetime, int nnIndex, int numNNs, String blockPoolId,
- String encryptionAlgorithm) {
- this(true, keyUpdateInterval, tokenLifetime, blockPoolId, encryptionAlgorithm, nnIndex, numNNs);
+ String encryptionAlgorithm, boolean useProto) {
+ this(true, keyUpdateInterval, tokenLifetime, blockPoolId,
+ encryptionAlgorithm, nnIndex, numNNs, useProto);
Preconditions.checkArgument(nnIndex >= 0);
Preconditions.checkArgument(numNNs > 0);
setSerialNo(new SecureRandom().nextInt());
@@ -111,7 +115,8 @@ public class BlockTokenSecretManager extends
}
private BlockTokenSecretManager(boolean isMaster, long keyUpdateInterval,
- long tokenLifetime, String blockPoolId, String encryptionAlgorithm, int nnIndex, int numNNs) {
+ long tokenLifetime, String blockPoolId, String encryptionAlgorithm,
+ int nnIndex, int numNNs, boolean useProto) {
this.intRange = Integer.MAX_VALUE / numNNs;
this.nnRangeStart = intRange * nnIndex;
this.isMaster = isMaster;
@@ -120,6 +125,7 @@ public class BlockTokenSecretManager extends
this.allKeys = new HashMap<Integer, BlockKey>();
this.blockPoolId = blockPoolId;
this.encryptionAlgorithm = encryptionAlgorithm;
+ this.useProto = useProto;
generateKeys();
}
@@ -246,7 +252,7 @@ public class BlockTokenSecretManager extends
public Token<BlockTokenIdentifier> generateToken(String userId,
ExtendedBlock block, EnumSet<BlockTokenIdentifier.AccessMode> modes) throws IOException {
BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
- .getBlockPoolId(), block.getBlockId(), modes);
+ .getBlockPoolId(), block.getBlockId(), modes, useProto);
return new Token<BlockTokenIdentifier>(id, this);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
index 1c6b352..0aa6fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
@@ -69,8 +69,12 @@ public class KeyManager implements Closeable, DataEncryptionKeyFactory {
+ ", token lifetime=" + StringUtils.formatTime(tokenLifetime));
String encryptionAlgorithm = conf.get(
DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
+ final boolean enableProtobuf = conf.getBoolean(
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE,
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE_DEFAULT);
this.blockTokenSecretManager = new BlockTokenSecretManager(
- updateInterval, tokenLifetime, blockpoolID, encryptionAlgorithm);
+ updateInterval, tokenLifetime, blockpoolID, encryptionAlgorithm,
+ enableProtobuf);
this.blockTokenSecretManager.addKeys(keys);
// sync block keys with NN more frequently than NN updates its block keys
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 893b12d..5125b33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -542,6 +542,9 @@ public class BlockManager implements BlockStatsMXBean {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
boolean isHaEnabled = HAUtil.isHAEnabled(conf, nsId);
+ boolean shouldWriteProtobufToken = conf.getBoolean(
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE,
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE_DEFAULT);
if (isHaEnabled) {
// figure out which index we are of the nns
@@ -555,10 +558,12 @@ public class BlockManager implements BlockStatsMXBean {
nnIndex++;
}
return new BlockTokenSecretManager(updateMin * 60 * 1000L,
- lifetimeMin * 60 * 1000L, nnIndex, nnIds.size(), null, encryptionAlgorithm);
+ lifetimeMin * 60 * 1000L, nnIndex, nnIds.size(), null,
+ encryptionAlgorithm, shouldWriteProtobufToken);
} else {
return new BlockTokenSecretManager(updateMin*60*1000L,
- lifetimeMin*60*1000L, 0, 1, null, encryptionAlgorithm);
+ lifetimeMin*60*1000L, 0, 1, null, encryptionAlgorithm,
+ shouldWriteProtobufToken);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index a6dfa46..9ed80ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1534,9 +1534,12 @@ public class DataNode extends ReconfigurableBase
+ blockKeyUpdateInterval / (60 * 1000)
+ " min(s), tokenLifetime=" + blockTokenLifetime / (60 * 1000)
+ " min(s)");
+ final boolean enableProtobuf = getConf().getBoolean(
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE,
+ DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE_DEFAULT);
final BlockTokenSecretManager secretMgr =
new BlockTokenSecretManager(0, blockTokenLifetime, blockPoolId,
- dnConf.encryptionAlgorithm);
+ dnConf.encryptionAlgorithm, enableProtobuf);
blockPoolTokenSecretManager.addBlockPool(blockPoolId, secretMgr);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 2bbc788..03f1a08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -503,6 +503,15 @@
</property>
<property>
+ <name>dfs.block.access.token.protobuf.enable</name>
+ <value>false</value>
+ <description>
+ If "true", block tokens are written using Protocol Buffers.
+ If "false", block tokens are written using Legacy format.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.data.dir</name>
<value>file://${hadoop.tmp.dir}/dfs/data</value>
<description>Determines where on the local filesystem an DFS data node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ed33e9c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
index 55e9d30..ecb63ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.security.token.block;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
@@ -31,7 +32,10 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Calendar;
import java.util.EnumSet;
+import java.util.GregorianCalendar;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -57,6 +61,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetRep
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto;
import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.TestWritable;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -104,7 +110,7 @@ public class TestBlockToken {
final ExtendedBlock block1 = new ExtendedBlock("0", 0L);
final ExtendedBlock block2 = new ExtendedBlock("10", 10L);
final ExtendedBlock block3 = new ExtendedBlock("-10", -108L);
-
+
@Before
public void disableKerberos() {
Configuration conf = new Configuration();
@@ -128,7 +134,7 @@ public class TestBlockToken {
InvocationOnMock invocation) throws IOException {
Object args[] = invocation.getArguments();
assertEquals(2, args.length);
- GetReplicaVisibleLengthRequestProto req =
+ GetReplicaVisibleLengthRequestProto req =
(GetReplicaVisibleLengthRequestProto) args[1];
Set<TokenIdentifier> tokenIds = UserGroupInformation.getCurrentUser()
.getTokenIdentifiers();
@@ -158,11 +164,11 @@ public class TestBlockToken {
return id;
}
- @Test
- public void testWritable() throws Exception {
+ private void testWritable(boolean enableProtobuf) throws Exception {
TestWritable.testWritable(new BlockTokenIdentifier());
BlockTokenSecretManager sm = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ enableProtobuf);
TestWritable.testWritable(generateTokenId(sm, block1,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class)));
TestWritable.testWritable(generateTokenId(sm, block2,
@@ -171,6 +177,16 @@ public class TestBlockToken {
EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class)));
}
+ @Test
+ public void testWritableLegacy() throws Exception {
+ testWritable(false);
+ }
+
+ @Test
+ public void testWritableProtobuf() throws Exception {
+ testWritable(true);
+ }
+
private void tokenGenerationAndVerification(BlockTokenSecretManager master,
BlockTokenSecretManager slave) throws Exception {
// single-mode tokens
@@ -198,12 +214,14 @@ public class TestBlockToken {
}
/** test block key and token handling */
- @Test
- public void testBlockTokenSecretManager() throws Exception {
+ private void testBlockTokenSecretManager(boolean enableProtobuf)
+ throws Exception {
BlockTokenSecretManager masterHandler = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ enableProtobuf);
BlockTokenSecretManager slaveHandler = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, "fake-pool", null,
+ enableProtobuf);
ExportedBlockKeys keys = masterHandler.exportKeys();
slaveHandler.addKeys(keys);
tokenGenerationAndVerification(masterHandler, slaveHandler);
@@ -215,6 +233,16 @@ public class TestBlockToken {
tokenGenerationAndVerification(masterHandler, slaveHandler);
}
+ @Test
+ public void testBlockTokenSecretManagerLegacy() throws Exception {
+ testBlockTokenSecretManager(false);
+ }
+
+ @Test
+ public void testBlockTokenSecretManagerProtobuf() throws Exception {
+ testBlockTokenSecretManager(true);
+ }
+
private static Server createMockDatanode(BlockTokenSecretManager sm,
Token<BlockTokenIdentifier> token, Configuration conf)
throws IOException, ServiceException {
@@ -223,7 +251,7 @@ public class TestBlockToken {
BlockTokenIdentifier id = sm.createIdentifier();
id.readFields(new DataInputStream(new ByteArrayInputStream(token
.getIdentifier())));
-
+
doAnswer(new GetLengthAnswer(sm, id)).when(mockDN)
.getReplicaVisibleLength(any(RpcController.class),
any(GetReplicaVisibleLengthRequestProto.class));
@@ -237,14 +265,14 @@ public class TestBlockToken {
.setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
}
- @Test
- public void testBlockTokenRpc() throws Exception {
+ private void testBlockTokenRpc(boolean enableProtobuf) throws Exception {
Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
-
+
BlockTokenSecretManager sm = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ enableProtobuf);
Token<BlockTokenIdentifier> token = sm.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class));
@@ -270,20 +298,30 @@ public class TestBlockToken {
}
}
+ @Test
+ public void testBlockTokenRpcLegacy() throws Exception {
+ testBlockTokenRpc(false);
+ }
+
+ @Test
+ public void testBlockTokenRpcProtobuf() throws Exception {
+ testBlockTokenRpc(true);
+ }
+
/**
* Test that fast repeated invocations of createClientDatanodeProtocolProxy
* will not end up using up thousands of sockets. This is a regression test
* for HDFS-1965.
*/
- @Test
- public void testBlockTokenRpcLeak() throws Exception {
+ private void testBlockTokenRpcLeak(boolean enableProtobuf) throws Exception {
Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
-
+
Assume.assumeTrue(FD_DIR.exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ enableProtobuf);
Token<BlockTokenIdentifier> token = sm.generateToken(block3,
EnumSet.allOf(BlockTokenIdentifier.AccessMode.class));
@@ -334,6 +372,16 @@ public class TestBlockToken {
RPC.stopProxy(proxyToNoWhere);
}
+ @Test
+ public void testBlockTokenRpcLeakLegacy() throws Exception {
+ testBlockTokenRpcLeak(false);
+ }
+
+ @Test
+ public void testBlockTokenRpcLeakProtobuf() throws Exception {
+ testBlockTokenRpcLeak(true);
+ }
+
/**
* @return the current number of file descriptors open by this process.
*/
@@ -344,17 +392,19 @@ public class TestBlockToken {
/**
* Test {@link BlockPoolTokenSecretManager}
*/
- @Test
- public void testBlockPoolTokenSecretManager() throws Exception {
+ private void testBlockPoolTokenSecretManager(boolean enableProtobuf)
+ throws Exception {
BlockPoolTokenSecretManager bpMgr = new BlockPoolTokenSecretManager();
// Test BlockPoolSecretManager with upto 10 block pools
for (int i = 0; i < 10; i++) {
String bpid = Integer.toString(i);
BlockTokenSecretManager masterHandler = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ enableProtobuf);
BlockTokenSecretManager slaveHandler = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, "fake-pool", null);
+ blockKeyUpdateInterval, blockTokenLifetime, "fake-pool", null,
+ enableProtobuf);
bpMgr.addBlockPool(bpid, slaveHandler);
ExportedBlockKeys keys = masterHandler.exportKeys();
@@ -370,20 +420,31 @@ public class TestBlockToken {
}
}
+ @Test
+ public void testBlockPoolTokenSecretManagerLegacy() throws Exception {
+ testBlockPoolTokenSecretManager(false);
+ }
+
+ @Test
+ public void testBlockPoolTokenSecretManagerProtobuf() throws Exception {
+ testBlockPoolTokenSecretManager(true);
+ }
+
/**
* This test writes a file and gets the block locations without closing the
* file, and tests the block token in the last block. Block token is verified
* by ensuring it is of correct kind.
- *
+ *
* @throws IOException
* @throws InterruptedException
*/
- @Test
- public void testBlockTokenInLastLocatedBlock() throws IOException,
- InterruptedException {
+ private void testBlockTokenInLastLocatedBlock(boolean enableProtobuf)
+ throws IOException, InterruptedException {
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
+ conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE,
+ enableProtobuf);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(1).build();
cluster.waitActive();
@@ -411,4 +472,188 @@ public class TestBlockToken {
cluster.shutdown();
}
}
+
+ @Test
+ public void testBlockTokenInLastLocatedBlockLegacy() throws IOException,
+ InterruptedException {
+ testBlockTokenInLastLocatedBlock(false);
+ }
+
+ @Test
+ public void testBlockTokenInLastLocatedBlockProtobuf() throws IOException,
+ InterruptedException {
+ testBlockTokenInLastLocatedBlock(true);
+ }
+
+ @Test
+ public void testLegacyBlockTokenBytesIsLegacy() throws IOException {
+ final boolean useProto = false;
+ BlockTokenSecretManager sm = new BlockTokenSecretManager(
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ useProto);
+ Token<BlockTokenIdentifier> token = sm.generateToken(block1,
+ EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class));
+ final byte[] tokenBytes = token.getIdentifier();
+ BlockTokenIdentifier legacyToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier protobufToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier readToken = new BlockTokenIdentifier();
+
+ DataInputBuffer dib = new DataInputBuffer();
+
+ dib.reset(tokenBytes, tokenBytes.length);
+ legacyToken.readFieldsLegacy(dib);
+
+ boolean invalidProtobufMessage = false;
+ try {
+ dib.reset(tokenBytes, tokenBytes.length);
+ protobufToken.readFieldsProtobuf(dib);
+ } catch (IOException e) {
+ invalidProtobufMessage = true;
+ }
+ assertTrue(invalidProtobufMessage);
+
+ dib.reset(tokenBytes, tokenBytes.length);
+ readToken.readFields(dib);
+
+ // Using legacy, the token parses as a legacy block token and not a protobuf
+ assertEquals(legacyToken, readToken);
+ assertNotEquals(protobufToken, readToken);
+ }
+
+ @Test
+ public void testEmptyLegacyBlockTokenBytesIsLegacy() throws IOException {
+ BlockTokenIdentifier emptyIdent = new BlockTokenIdentifier();
+ DataOutputBuffer dob = new DataOutputBuffer(4096);
+ DataInputBuffer dib = new DataInputBuffer();
+
+ emptyIdent.writeLegacy(dob);
+ byte[] emptyIdentBytes = Arrays.copyOf(dob.getData(), dob.getLength());
+
+ BlockTokenIdentifier legacyToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier protobufToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier readToken = new BlockTokenIdentifier();
+
+ dib.reset(emptyIdentBytes, emptyIdentBytes.length);
+ legacyToken.readFieldsLegacy(dib);
+
+ boolean invalidProtobufMessage = false;
+ try {
+ dib.reset(emptyIdentBytes, emptyIdentBytes.length);
+ protobufToken.readFieldsProtobuf(dib);
+ } catch (IOException e) {
+ invalidProtobufMessage = true;
+ }
+ assertTrue(invalidProtobufMessage);
+
+ dib.reset(emptyIdentBytes, emptyIdentBytes.length);
+ readToken.readFields(dib);
+ assertTrue(invalidProtobufMessage);
+ }
+
+ @Test
+ public void testProtobufBlockTokenBytesIsProtobuf() throws IOException {
+ final boolean useProto = true;
+ BlockTokenSecretManager sm = new BlockTokenSecretManager(
+ blockKeyUpdateInterval, blockTokenLifetime, 0, 1, "fake-pool", null,
+ useProto);
+ Token<BlockTokenIdentifier> token = sm.generateToken(block1,
+ EnumSet.noneOf(BlockTokenIdentifier.AccessMode.class));
+ final byte[] tokenBytes = token.getIdentifier();
+ BlockTokenIdentifier legacyToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier protobufToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier readToken = new BlockTokenIdentifier();
+
+ DataInputBuffer dib = new DataInputBuffer();
+
+ /* We receive NegativeArraySizeException because we didn't call
+ * readFields and instead try to parse this directly as a legacy
+ * BlockTokenIdentifier.
+ *
+ * Note: because the parsing depends on the expiryDate which is based on
+ * `Time.now()` it can sometimes fail with IOException and sometimes with
+ * NegativeArraySizeException.
+ */
+ boolean invalidLegacyMessage = false;
+ try {
+ dib.reset(tokenBytes, tokenBytes.length);
+ legacyToken.readFieldsLegacy(dib);
+ } catch (IOException | NegativeArraySizeException e) {
+ invalidLegacyMessage = true;
+ }
+ assertTrue(invalidLegacyMessage);
+
+ dib.reset(tokenBytes, tokenBytes.length);
+ protobufToken.readFieldsProtobuf(dib);
+
+ dib.reset(tokenBytes, tokenBytes.length);
+ readToken.readFields(dib);
+
+ // Using protobuf, the token parses as a protobuf and not a legacy block
+ // token
+ assertNotEquals(legacyToken, readToken);
+ assertEquals(protobufToken, readToken);
+ }
+
+ public void testCraftedProtobufBlockTokenIdentifier(
+ BlockTokenIdentifier identifier, boolean expectIOE,
+ boolean expectRTE) throws IOException {
+ DataOutputBuffer dob = new DataOutputBuffer(4096);
+ DataInputBuffer dib = new DataInputBuffer();
+
+ identifier.writeProtobuf(dob);
+ byte[] identBytes = Arrays.copyOf(dob.getData(), dob.getLength());
+
+ BlockTokenIdentifier legacyToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier protobufToken = new BlockTokenIdentifier();
+ BlockTokenIdentifier readToken = new BlockTokenIdentifier();
+
+ boolean invalidLegacyMessage = false;
+ try {
+ dib.reset(identBytes, identBytes.length);
+ legacyToken.readFieldsLegacy(dib);
+ } catch (IOException e) {
+ if (!expectIOE) {
+ fail("Received IOException but it was not expected.");
+ }
+ invalidLegacyMessage = true;
+ } catch (RuntimeException e) {
+ if (!expectRTE) {
+ fail("Received RuntimeException but it was not expected.");
+ }
+ invalidLegacyMessage = true;
+ }
+
+ assertTrue(invalidLegacyMessage);
+
+ dib.reset(identBytes, identBytes.length);
+ protobufToken.readFieldsProtobuf(dib);
+
+ dib.reset(identBytes, identBytes.length);
+ readToken.readFieldsProtobuf(dib);
+ assertEquals(protobufToken, readToken);
+ }
+
+ @Test
+ public void testCraftedProtobufBlockTokenBytesIsProtobuf() throws
+ IOException {
+ // Empty BlockTokenIdentifiers throw IOException
+ BlockTokenIdentifier identifier = new BlockTokenIdentifier();
+ testCraftedProtobufBlockTokenIdentifier(identifier, true, false);
+
+ /* Parsing BlockTokenIdentifier with expiryDate
+ * 2017-02-09 00:12:35,072+0100 will throw IOException.
+ * However, expiryDate of
+ * 2017-02-09 00:12:35,071+0100 will throw NegativeArraySizeException.
+ */
+ Calendar cal = new GregorianCalendar();
+ cal.set(2017, 1, 9, 0, 12, 35);
+ long datetime = cal.getTimeInMillis();
+ datetime = ((datetime / 1000) * 1000); // strip milliseconds.
+ datetime = datetime + 71; // 2017-02-09 00:12:35,071+0100
+ identifier.setExpiryDate(datetime);
+ testCraftedProtobufBlockTokenIdentifier(identifier, false, true);
+ datetime += 1; // 2017-02-09 00:12:35,072+0100
+ identifier.setExpiryDate(datetime);
+ testCraftedProtobufBlockTokenIdentifier(identifier, true, false);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-3664. Federation PolicyStore
internal APIs
Posted by su...@apache.org.
YARN-3664. Federation PolicyStore internal APIs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edbcd9bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edbcd9bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edbcd9bd
Branch: refs/heads/YARN-2915
Commit: edbcd9bd40e0593dabee3df0ac4d2f75a605f064
Parents: 4c50328
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Aug 5 12:34:58 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../federation/store/FederationPolicyStore.java | 76 ++++++++
...SubClusterPoliciesConfigurationsRequest.java | 35 ++++
...ubClusterPoliciesConfigurationsResponse.java | 66 +++++++
...GetSubClusterPolicyConfigurationRequest.java | 62 ++++++
...etSubClusterPolicyConfigurationResponse.java | 65 +++++++
...SetSubClusterPolicyConfigurationRequest.java | 79 ++++++++
...etSubClusterPolicyConfigurationResponse.java | 36 ++++
.../records/SubClusterPolicyConfiguration.java | 130 +++++++++++++
...sterPoliciesConfigurationsRequestPBImpl.java | 95 +++++++++
...terPoliciesConfigurationsResponsePBImpl.java | 191 +++++++++++++++++++
...ClusterPolicyConfigurationRequestPBImpl.java | 103 ++++++++++
...lusterPolicyConfigurationResponsePBImpl.java | 143 ++++++++++++++
.../pb/GetSubClustersInfoResponsePBImpl.java | 4 +-
...ClusterPolicyConfigurationRequestPBImpl.java | 159 +++++++++++++++
...lusterPolicyConfigurationResponsePBImpl.java | 93 +++++++++
.../pb/SubClusterPolicyConfigurationPBImpl.java | 121 ++++++++++++
.../proto/yarn_server_federation_protos.proto | 28 +++
.../records/TestFederationProtocolRecords.java | 53 ++++-
18 files changed, 1536 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
new file mode 100644
index 0000000..9d9bd9b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+
+/**
+ * The FederationPolicyStore provides a key-value interface to access the
+ * policies configured for the system. The key is a "queue" name, i.e., the
+ * system allows to configure a different policy for each queue in the system
+ * (though each policy can make dynamic run-time decisions on a per-job/per-task
+ * basis). The value is a {@code SubClusterPolicyConfiguration}, a serialized
+ * representation of the policy type and its parameters.
+ */
+@Private
+@Unstable
+public interface FederationPolicyStore {
+
+ /**
+ * Get the policy configuration for a given queue.
+ *
+ * @param request the queue whose {@code SubClusterPolicyConfiguration} is
+ * required
+ * @return the {@code SubClusterPolicyConfiguration} for the specified queue
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException;
+
+ /**
+ * Set the policy configuration for a given queue.
+ *
+ * @param request the {@code SubClusterPolicyConfiguration} with the
+ * corresponding queue
+ * @return response empty on successfully updating the
+ * {@code SubClusterPolicyConfiguration} for the specified queue
+ * @throws YarnException if the request is invalid/fails
+ */
+ SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException;
+
+ /**
+ * Get a map of all queue-to-policy configurations.
+ *
+ * @param request empty to represent all configured queues in the system
+ * @return the policies for all currently active queues in the system
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
new file mode 100644
index 0000000..404521b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPoliciesConfigurationsRequest is a request to the
+ * {@code FederationPolicyStore} to obtain all policy configurations.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPoliciesConfigurationsRequest {
+ public GetSubClusterPoliciesConfigurationsRequest newInstance() {
+ return Records.newRecord(GetSubClusterPoliciesConfigurationsRequest.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
new file mode 100644
index 0000000..6554d68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPolicyConfigurationResponse contains the answer from the {@code
+ * FederationPolicyStore} to a request to get all the policies configured in the
+ * system via a {@link SubClusterPolicyConfiguration}.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPoliciesConfigurationsResponse {
+
+ @Private
+ @Unstable
+ public GetSubClusterPoliciesConfigurationsResponse newInstance(
+ List<SubClusterPolicyConfiguration> policyConfigurations) {
+ GetSubClusterPoliciesConfigurationsResponse response =
+ Records.newRecord(GetSubClusterPoliciesConfigurationsResponse.class);
+ response.setPoliciesConfigs(policyConfigurations);
+ return response;
+ }
+
+ /**
+ * Get all the policies configured in the system.
+ *
+ * @return all the policies configured in the system
+ */
+ @Public
+ @Unstable
+ public abstract List<SubClusterPolicyConfiguration> getPoliciesConfigs();
+
+ /**
+ * Sets all the policies configured in the system.
+ *
+ * @param policyConfigurations all the policies configured in the system
+ */
+ @Private
+ @Unstable
+ public abstract void setPoliciesConfigs(
+ List<SubClusterPolicyConfiguration> policyConfigurations);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
new file mode 100644
index 0000000..7b7d8c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPolicyConfigurationRequest is a request to the
+ * {@code FederationPolicyStore} to get the configuration of a policy for a
+ * given queue.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPolicyConfigurationRequest {
+
+ @Private
+ @Unstable
+ public GetSubClusterPolicyConfigurationRequest newInstance(String queueName) {
+ GetSubClusterPolicyConfigurationRequest request =
+ Records.newRecord(GetSubClusterPolicyConfigurationRequest.class);
+ request.setQueue(queueName);
+ return request;
+ }
+
+ /**
+ * Get the name of the queue for which we are requesting a policy
+ * configuration.
+ *
+ * @return the name of the queue
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Sets the name of the queue for which we are requesting a policy
+ * configuration.
+ *
+ * @param queueName the name of the queue
+ */
+ @Private
+ @Unstable
+ public abstract void setQueue(String queueName);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
new file mode 100644
index 0000000..11a46e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPolicyConfigurationResponse contains the answer from the {@code
+ * FederationPolicyStore} to a request to get the information about how a policy
+ * should be configured via a {@link SubClusterPolicyConfiguration}.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPolicyConfigurationResponse {
+
+ @Private
+ @Unstable
+ public GetSubClusterPolicyConfigurationResponse newInstance(
+ SubClusterPolicyConfiguration policy) {
+ GetSubClusterPolicyConfigurationResponse response =
+ Records.newRecord(GetSubClusterPolicyConfigurationResponse.class);
+ response.setPolicyConfiguration(policy);
+ return response;
+ }
+
+ /**
+ * Get the policy configuration.
+ *
+ * @return the policy configuration for the specified queue
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterPolicyConfiguration getPolicyConfiguration();
+
+ /**
+ * Sets the policyConfiguration configuration.
+ *
+ * @param policyConfiguration the policyConfiguration configuration for the
+ * specified queue
+ */
+ @Private
+ @Unstable
+ public abstract void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
new file mode 100644
index 0000000..06d5399
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SetSubClusterPolicyConfigurationRequest is a request to the
+ * {@code FederationPolicyStore} to set the policy configuration corresponding
+ * to a queue.
+ */
+@Private
+@Unstable
+public abstract class SetSubClusterPolicyConfigurationRequest {
+ @Private
+ @Unstable
+ public SetSubClusterPolicyConfigurationRequest newInstance(
+ SubClusterPolicyConfiguration policy) {
+ SetSubClusterPolicyConfigurationRequest request =
+ Records.newRecord(SetSubClusterPolicyConfigurationRequest.class);
+ request.setPolicyConfiguration(policy);
+ return request;
+ }
+
+ /**
+ * Get the name of the queue for which we are configuring a policy.
+ *
+ * @return the name of the queue
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Sets the name of the queue for which we are configuring a policy.
+ *
+ * @param queueName the name of the queue
+ */
+ @Private
+ @Unstable
+ public abstract void setQueue(String queueName);
+
+ /**
+ * Get the policy configuration assigned to the queue.
+ *
+ * @return the policy for the specified queue
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterPolicyConfiguration getPolicyConfiguration();
+
+ /**
+ * Set the policyConfiguration configuration for the queue.
+ *
+ * @param policyConfiguration the policyConfiguration for the specified queue
+ */
+ @Private
+ @Unstable
+ public abstract void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
new file mode 100644
index 0000000..33c4043
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SetSubClusterPolicyConfigurationResponse contains the answer from the
+ * {@code FederationPolicyStore} to a request to set for a policy configuration
+ * for a given queue.
+ */
+@Private
+@Unstable
+public abstract class SetSubClusterPolicyConfigurationResponse {
+ public SetSubClusterPolicyConfigurationResponse newInstance() {
+ return Records.newRecord(SetSubClusterPolicyConfigurationResponse.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
new file mode 100644
index 0000000..bc12acb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.nio.ByteBuffer;
+
+// used in javadoc
+
+/**
+ * {@link SubClusterPolicyConfiguration} is a class that represents a
+ * configuration of a policy. It contains a policy type (resolve to a class
+ * name) and its params as an opaque {@link ByteBuffer}.
+ *
+ * Note: by design the params are an opaque ByteBuffer, this allows for enough
+ * flexibility to evolve the policies without impacting the protocols to/from
+ * the federation state store.
+ */
+@Private
+@Unstable
+public abstract class SubClusterPolicyConfiguration {
+
+ @Private
+ @Unstable
+ public static SubClusterPolicyConfiguration newInstance(String policyType,
+ ByteBuffer policyParams) {
+ SubClusterPolicyConfiguration policy =
+ Records.newRecord(SubClusterPolicyConfiguration.class);
+ policy.setType(policyType);
+ policy.setParams(policyParams);
+ return policy;
+ }
+
+ /**
+ * Get the type of the policy. This could be random, round-robin, load-based,
+ * etc.
+ *
+ * @return the type of the policy
+ */
+ @Public
+ @Unstable
+ public abstract String getType();
+
+ /**
+ * Sets the type of the policy. This could be random, round-robin, load-based,
+ * etc.
+ *
+ * @param policyType the type of the policy
+ */
+ @Private
+ @Unstable
+ public abstract void setType(String policyType);
+
+ /**
+ * Get the policy parameters. This affects how the policy behaves and an
+ * example could be weight distribution of queues across multiple
+ * sub-clusters.
+ *
+ * @return the byte array that contains the parameters
+ */
+ @Public
+ @Unstable
+ public abstract ByteBuffer getParams();
+
+ /**
+ * Set the policy parameters. This affects how the policy behaves and an
+ * example could be weight distribution of queues across multiple
+ * sub-clusters.
+ *
+ * @param policyParams byte array that describes the policy
+ */
+ @Private
+ @Unstable
+ public abstract void setParams(ByteBuffer policyParams);
+
+ @Override
+ public int hashCode() {
+ return 31 * getParams().hashCode() + getType().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ SubClusterPolicyConfiguration other = (SubClusterPolicyConfiguration) obj;
+ if (!this.getType().equals(other.getType())) {
+ return false;
+ }
+ if (!this.getParams().equals(other.getParams())) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getType());
+ sb.append(" : ");
+ sb.append(getParams());
+ return sb.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java
new file mode 100644
index 0000000..3cb6601
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsRequestProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPoliciesConfigurationsRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPoliciesConfigurationsRequestPBImpl
+ extends GetSubClusterPoliciesConfigurationsRequest {
+
+ private GetSubClusterPoliciesConfigurationsRequestProto proto =
+ GetSubClusterPoliciesConfigurationsRequestProto.getDefaultInstance();
+ private GetSubClusterPoliciesConfigurationsRequestProto.Builder builder =
+ null;
+ private boolean viaProto = false;
+
+ public GetSubClusterPoliciesConfigurationsRequestPBImpl() {
+ builder = GetSubClusterPoliciesConfigurationsRequestProto.newBuilder();
+ }
+
+ public GetSubClusterPoliciesConfigurationsRequestPBImpl(
+ GetSubClusterPoliciesConfigurationsRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPoliciesConfigurationsRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder =
+ GetSubClusterPoliciesConfigurationsRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
new file mode 100644
index 0000000..67c3654
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPoliciesConfigurationsResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPoliciesConfigurationsResponsePBImpl
+ extends GetSubClusterPoliciesConfigurationsResponse {
+
+ private GetSubClusterPoliciesConfigurationsResponseProto proto =
+ GetSubClusterPoliciesConfigurationsResponseProto.getDefaultInstance();
+ private GetSubClusterPoliciesConfigurationsResponseProto.Builder builder =
+ null;
+ private boolean viaProto = false;
+
+ private List<SubClusterPolicyConfiguration> subClusterPolicies = null;
+
+ public GetSubClusterPoliciesConfigurationsResponsePBImpl() {
+ builder = GetSubClusterPoliciesConfigurationsResponseProto.newBuilder();
+ }
+
+ public GetSubClusterPoliciesConfigurationsResponsePBImpl(
+ GetSubClusterPoliciesConfigurationsResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPoliciesConfigurationsResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder =
+ GetSubClusterPoliciesConfigurationsResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterPolicies != null) {
+ addSubClusterPoliciesConfigurationsToProto();
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public List<SubClusterPolicyConfiguration> getPoliciesConfigs() {
+ initSubClusterPoliciesConfigurationsList();
+ return this.subClusterPolicies;
+ }
+
+ @Override
+ public void setPoliciesConfigs(
+ List<SubClusterPolicyConfiguration> policyConfigurations) {
+ maybeInitBuilder();
+ if (policyConfigurations == null) {
+ builder.clearPoliciesConfigurations();
+ }
+ this.subClusterPolicies = policyConfigurations;
+ }
+
+ private void initSubClusterPoliciesConfigurationsList() {
+ if (this.subClusterPolicies != null) {
+ return;
+ }
+ GetSubClusterPoliciesConfigurationsResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<SubClusterPolicyConfigurationProto> subClusterPoliciesList =
+ p.getPoliciesConfigurationsList();
+ subClusterPolicies = new ArrayList<SubClusterPolicyConfiguration>();
+
+ for (SubClusterPolicyConfigurationProto r : subClusterPoliciesList) {
+ subClusterPolicies.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addSubClusterPoliciesConfigurationsToProto() {
+ maybeInitBuilder();
+ builder.clearPoliciesConfigurations();
+ if (subClusterPolicies == null) {
+ return;
+ }
+ Iterable<SubClusterPolicyConfigurationProto> iterable =
+ new Iterable<SubClusterPolicyConfigurationProto>() {
+ @Override
+ public Iterator<SubClusterPolicyConfigurationProto> iterator() {
+ return new Iterator<SubClusterPolicyConfigurationProto>() {
+
+ private Iterator<SubClusterPolicyConfiguration> iter =
+ subClusterPolicies.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public SubClusterPolicyConfigurationProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllPoliciesConfigurations(iterable);
+ }
+
+ private SubClusterPolicyConfiguration convertFromProtoFormat(
+ SubClusterPolicyConfigurationProto policy) {
+ return new SubClusterPolicyConfigurationPBImpl(policy);
+ }
+
+ private SubClusterPolicyConfigurationProto convertToProtoFormat(
+ SubClusterPolicyConfiguration policy) {
+ return ((SubClusterPolicyConfigurationPBImpl) policy).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java
new file mode 100644
index 0000000..35aff17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPolicyConfigurationRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPolicyConfigurationRequestPBImpl
+ extends GetSubClusterPolicyConfigurationRequest {
+
+ private GetSubClusterPolicyConfigurationRequestProto proto =
+ GetSubClusterPolicyConfigurationRequestProto.getDefaultInstance();
+ private GetSubClusterPolicyConfigurationRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetSubClusterPolicyConfigurationRequestPBImpl() {
+ builder = GetSubClusterPolicyConfigurationRequestProto.newBuilder();
+ }
+
+ public GetSubClusterPolicyConfigurationRequestPBImpl(
+ GetSubClusterPolicyConfigurationRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPolicyConfigurationRequestProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterPolicyConfigurationRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public String getQueue() {
+ GetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearQueue();
+ return;
+ }
+ builder.setQueue(queueName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
new file mode 100644
index 0000000..96b88ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPolicyConfigurationResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPolicyConfigurationResponsePBImpl
+ extends GetSubClusterPolicyConfigurationResponse {
+
+ private GetSubClusterPolicyConfigurationResponseProto proto =
+ GetSubClusterPolicyConfigurationResponseProto.getDefaultInstance();
+ private GetSubClusterPolicyConfigurationResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterPolicyConfiguration subClusterPolicy = null;
+
+ public GetSubClusterPolicyConfigurationResponsePBImpl() {
+ builder = GetSubClusterPolicyConfigurationResponseProto.newBuilder();
+ }
+
+ public GetSubClusterPolicyConfigurationResponsePBImpl(
+ GetSubClusterPolicyConfigurationResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPolicyConfigurationResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterPolicyConfigurationResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterPolicy != null
+ && !((SubClusterPolicyConfigurationPBImpl) this.subClusterPolicy)
+ .getProto().equals(builder.getPolicyConfiguration())) {
+ builder
+ .setPolicyConfiguration(convertToProtoFormat(this.subClusterPolicy));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration getPolicyConfiguration() {
+ GetSubClusterPolicyConfigurationResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.subClusterPolicy != null) {
+ return this.subClusterPolicy;
+ }
+ if (!p.hasPolicyConfiguration()) {
+ return null;
+ }
+ this.subClusterPolicy = convertFromProtoFormat(p.getPolicyConfiguration());
+ return this.subClusterPolicy;
+ }
+
+ @Override
+ public void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration) {
+ maybeInitBuilder();
+ if (policyConfiguration == null) {
+ builder.clearPolicyConfiguration();
+ }
+ this.subClusterPolicy = policyConfiguration;
+ }
+
+ private SubClusterPolicyConfiguration convertFromProtoFormat(
+ SubClusterPolicyConfigurationProto policy) {
+ return new SubClusterPolicyConfigurationPBImpl(policy);
+ }
+
+ private SubClusterPolicyConfigurationProto convertToProtoFormat(
+ SubClusterPolicyConfiguration policy) {
+ return ((SubClusterPolicyConfigurationPBImpl) policy).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
index 92bdf06..2efa3b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -66,7 +66,7 @@ public class GetSubClustersInfoResponsePBImpl
private void mergeLocalToBuilder() {
if (this.subClusterInfos != null) {
- addReservationResourcesToProto();
+ addSubClusterInfosToProto();
}
}
@@ -114,7 +114,7 @@ public class GetSubClustersInfoResponsePBImpl
}
}
- private void addReservationResourcesToProto() {
+ private void addSubClusterInfosToProto() {
maybeInitBuilder();
builder.clearSubClusterInfos();
if (subClusterInfos == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
new file mode 100644
index 0000000..5e29bd5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link SetSubClusterPolicyConfigurationRequest}.
+ */
+@Private
+@Unstable
+public class SetSubClusterPolicyConfigurationRequestPBImpl
+ extends SetSubClusterPolicyConfigurationRequest {
+
+ private SetSubClusterPolicyConfigurationRequestProto proto =
+ SetSubClusterPolicyConfigurationRequestProto.getDefaultInstance();
+ private SetSubClusterPolicyConfigurationRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterPolicyConfiguration subClusterPolicy = null;
+
+ public SetSubClusterPolicyConfigurationRequestPBImpl() {
+ builder = SetSubClusterPolicyConfigurationRequestProto.newBuilder();
+ }
+
+ public SetSubClusterPolicyConfigurationRequestPBImpl(
+ SetSubClusterPolicyConfigurationRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SetSubClusterPolicyConfigurationRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SetSubClusterPolicyConfigurationRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterPolicy != null
+ && !((SubClusterPolicyConfigurationPBImpl) this.subClusterPolicy)
+ .getProto().equals(builder.getPolicyConfiguration())) {
+ builder
+ .setPolicyConfiguration(convertToProtoFormat(this.subClusterPolicy));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public String getQueue() {
+ SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearQueue();
+ return;
+ }
+ builder.setQueue(queueName);
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration getPolicyConfiguration() {
+ SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.subClusterPolicy != null) {
+ return this.subClusterPolicy;
+ }
+ if (!p.hasPolicyConfiguration()) {
+ return null;
+ }
+ this.subClusterPolicy = convertFromProtoFormat(p.getPolicyConfiguration());
+ return this.subClusterPolicy;
+ }
+
+ @Override
+ public void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration) {
+ maybeInitBuilder();
+ if (policyConfiguration == null) {
+ builder.clearPolicyConfiguration();
+ }
+ this.subClusterPolicy = policyConfiguration;
+ }
+
+ private SubClusterPolicyConfiguration convertFromProtoFormat(
+ SubClusterPolicyConfigurationProto policy) {
+ return new SubClusterPolicyConfigurationPBImpl(policy);
+ }
+
+ private SubClusterPolicyConfigurationProto convertToProtoFormat(
+ SubClusterPolicyConfiguration policy) {
+ return ((SubClusterPolicyConfigurationPBImpl) policy).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java
new file mode 100644
index 0000000..9d79215
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link SetSubClusterPolicyConfigurationResponse}.
+ */
+@Private
+@Unstable
+public class SetSubClusterPolicyConfigurationResponsePBImpl
+ extends SetSubClusterPolicyConfigurationResponse {
+
+ private SetSubClusterPolicyConfigurationResponseProto proto =
+ SetSubClusterPolicyConfigurationResponseProto.getDefaultInstance();
+ private SetSubClusterPolicyConfigurationResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SetSubClusterPolicyConfigurationResponsePBImpl() {
+ builder = SetSubClusterPolicyConfigurationResponseProto.newBuilder();
+ }
+
+ public SetSubClusterPolicyConfigurationResponsePBImpl(
+ SetSubClusterPolicyConfigurationResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SetSubClusterPolicyConfigurationResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SetSubClusterPolicyConfigurationResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
new file mode 100644
index 0000000..fe9d9db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protobuf based implementation of {@link SubClusterPolicyConfiguration}.
+ *
+ */
+@Private
+@Unstable
+public class SubClusterPolicyConfigurationPBImpl
+ extends SubClusterPolicyConfiguration {
+
+ private SubClusterPolicyConfigurationProto proto =
+ SubClusterPolicyConfigurationProto.getDefaultInstance();
+ private SubClusterPolicyConfigurationProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterPolicyConfigurationPBImpl() {
+ builder = SubClusterPolicyConfigurationProto.newBuilder();
+ }
+
+ public SubClusterPolicyConfigurationPBImpl(
+ SubClusterPolicyConfigurationProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterPolicyConfigurationProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterPolicyConfigurationProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public String getType() {
+ SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getType();
+ }
+
+ @Override
+ public void setType(String policyType) {
+ maybeInitBuilder();
+ if (policyType == null) {
+ builder.clearType();
+ return;
+ }
+ builder.setType(policyType);
+ }
+
+ @Override
+ public ByteBuffer getParams() {
+ SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
+ return ProtoUtils.convertFromProtoFormat(p.getParams());
+ }
+
+ @Override
+ public void setParams(ByteBuffer policyParams) {
+ maybeInitBuilder();
+ if (policyParams == null) {
+ builder.clearParams();
+ return;
+ }
+ builder.setParams(ProtoUtils.convertToProtoFormat(policyParams));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index b1ad310..3f1cee9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -134,3 +134,31 @@ message DeleteApplicationHomeSubClusterRequestProto {
message DeleteApplicationHomeSubClusterResponseProto {
}
+
+message SubClusterPolicyConfigurationProto {
+ optional string type = 1;
+ optional bytes params = 2;
+}
+
+message GetSubClusterPolicyConfigurationRequestProto {
+ optional string queue = 1;
+}
+
+message GetSubClusterPolicyConfigurationResponseProto {
+ optional SubClusterPolicyConfigurationProto policy_configuration = 1;
+}
+
+message SetSubClusterPolicyConfigurationRequestProto {
+ optional string queue = 1;
+ optional SubClusterPolicyConfigurationProto policy_configuration = 2;
+}
+
+message SetSubClusterPolicyConfigurationResponseProto {
+}
+
+message GetSubClusterPoliciesConfigurationsRequestProto {
+}
+
+message GetSubClusterPoliciesConfigurationsResponseProto {
+ repeated SubClusterPolicyConfigurationProto policies_configurations = 1;
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edbcd9bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
index 210a246..cf8cf71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -29,8 +29,14 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApp
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProto;
@@ -51,8 +57,14 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplica
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationsHomeSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPoliciesConfigurationsRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPoliciesConfigurationsResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPolicyConfigurationRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPolicyConfigurationResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SetSubClusterPolicyConfigurationRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SetSubClusterPolicyConfigurationResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterHeartbeatRequestPBImpl;
@@ -79,6 +91,7 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
generateByNewInstance(SubClusterId.class);
generateByNewInstance(SubClusterInfo.class);
generateByNewInstance(ApplicationHomeSubCluster.class);
+ generateByNewInstance(SubClusterPolicyConfiguration.class);
}
@Test
@@ -211,4 +224,42 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
DeleteApplicationHomeSubClusterResponseProto.class);
}
-}
\ No newline at end of file
+ @Test
+ public void testGetSubClusterPolicyConfigurationRequest() throws Exception {
+ validatePBImplRecord(GetSubClusterPolicyConfigurationRequestPBImpl.class,
+ GetSubClusterPolicyConfigurationRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterPolicyConfigurationResponse() throws Exception {
+ validatePBImplRecord(GetSubClusterPolicyConfigurationResponsePBImpl.class,
+ GetSubClusterPolicyConfigurationResponseProto.class);
+ }
+
+ @Test
+ public void testSetSubClusterPolicyConfigurationRequest() throws Exception {
+ validatePBImplRecord(SetSubClusterPolicyConfigurationRequestPBImpl.class,
+ SetSubClusterPolicyConfigurationRequestProto.class);
+ }
+
+ @Test
+ public void testSetSubClusterPolicyConfigurationResponse() throws Exception {
+ validatePBImplRecord(SetSubClusterPolicyConfigurationResponsePBImpl.class,
+ SetSubClusterPolicyConfigurationResponseProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterPoliciesConfigurationsRequest()
+ throws Exception {
+ validatePBImplRecord(GetSubClusterPoliciesConfigurationsRequestPBImpl.class,
+ GetSubClusterPoliciesConfigurationsRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterPoliciesConfigurationsResponse()
+ throws Exception {
+ validatePBImplRecord(
+ GetSubClusterPoliciesConfigurationsResponsePBImpl.class,
+ GetSubClusterPoliciesConfigurationsResponseProto.class);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-5966. AMRMClient changes to
support ExecutionType update. (asuresh)
Posted by su...@apache.org.
YARN-5966. AMRMClient changes to support ExecutionType update. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaf106fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaf106fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaf106fd
Branch: refs/heads/YARN-2915
Commit: aaf106fde35ec97e2e2ea4d7a67434038c4273ac
Parents: 4164a20
Author: Arun Suresh <as...@apache.org>
Authored: Tue Feb 14 06:08:27 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Tue Feb 14 06:09:10 2017 -0800
----------------------------------------------------------------------
.../yarn/api/records/UpdateContainerError.java | 19 +-
.../src/main/proto/yarn_service_protos.proto | 1 +
.../hadoop/yarn/client/api/AMRMClient.java | 33 +-
.../yarn/client/api/async/AMRMClientAsync.java | 33 +-
.../api/async/impl/AMRMClientAsyncImpl.java | 7 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 111 +++--
.../yarn/client/api/impl/TestAMRMClient.java | 60 ++-
.../api/impl/TestAMRMClientOnRMRestart.java | 8 +-
.../TestOpportunisticContainerAllocation.java | 400 +++++++++++++++++--
.../impl/pb/UpdateContainerErrorPBImpl.java | 16 +
.../server/resourcemanager/RMServerUtils.java | 14 +-
...pportunisticContainerAllocatorAMService.java | 5 +-
.../capacity/TestIncreaseAllocationExpirer.java | 4 +-
13 files changed, 587 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
index e7458cf..4d184cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerError.java
@@ -59,6 +59,22 @@ public abstract class UpdateContainerError {
public abstract void setReason(String reason);
/**
+ * Get current container version.
+ * @return Current container Version.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract int getCurrentContainerVersion();
+
+ /**
+ * Set current container version.
+ * @param currentVersion Current container version.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract void setCurrentContainerVersion(int currentVersion);
+
+ /**
* Get the {@code UpdateContainerRequest} that was not satisfiable.
* @return UpdateContainerRequest
*/
@@ -89,6 +105,7 @@ public abstract class UpdateContainerError {
@Override
public String toString() {
return "UpdateContainerError{reason=" + getReason() + ", "
+ + "currentVersion=" + getCurrentContainerVersion() + ", "
+ "req=" + getUpdateContainerRequest() + "}";
}
@@ -120,6 +137,6 @@ public abstract class UpdateContainerError {
} else if (!req.equals(other.getUpdateContainerRequest())) {
return false;
}
- return true;
+ return getCurrentContainerVersion() == other.getCurrentContainerVersion();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index df3c852..c6647c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -78,6 +78,7 @@ message UpdateContainerRequestProto {
message UpdateContainerErrorProto {
optional string reason = 1;
optional UpdateContainerRequestProto update_request = 2;
+ optional int32 current_container_version = 3;
}
message AllocateRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 52155f5..15d0065 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -33,17 +33,20 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.yarn.util.resource.Resources;
@InterfaceAudience.Public
@InterfaceStability.Stable
@@ -518,12 +521,38 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
* ResourceManager to change the existing resource allocation to the target
* resource allocation.
*
+ * @deprecated use
+ * {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+ *
* @param container The container returned from the last successful resource
* allocation or resource change
* @param capability The target resource capability of the container
*/
- public abstract void requestContainerResourceChange(
- Container container, Resource capability);
+ @Deprecated
+ public void requestContainerResourceChange(
+ Container container, Resource capability) {
+ Preconditions.checkNotNull(container, "Container cannot be null!!");
+ Preconditions.checkNotNull(capability,
+ "UpdateContainerRequest cannot be null!!");
+ requestContainerUpdate(container, UpdateContainerRequest.newInstance(
+ container.getVersion(), container.getId(),
+ Resources.fitsIn(capability, container.getResource()) ?
+ ContainerUpdateType.DECREASE_RESOURCE :
+ ContainerUpdateType.INCREASE_RESOURCE,
+ capability, null));
+ }
+
+ /**
+ * Request a container update before calling <code>allocate</code>.
+ * Any previous pending update request of the same container will be
+ * removed.
+ *
+ * @param container The container returned from the last successful resource
+ * allocation or update
+ * @param updateContainerRequest The <code>UpdateContainerRequest</code>.
+ */
+ public abstract void requestContainerUpdate(
+ Container container, UpdateContainerRequest updateContainerRequest);
/**
* Release containers assigned by the Resource Manager. If the app cannot use
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index d2195a6..4cb27cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -36,11 +36,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -50,6 +52,7 @@ import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.util.resource.Resources;
/**
* <code>AMRMClientAsync</code> handles communication with the ResourceManager
@@ -284,12 +287,38 @@ extends AbstractService {
* ResourceManager to change the existing resource allocation to the target
* resource allocation.
*
+ * @deprecated use
+ * {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+ *
* @param container The container returned from the last successful resource
* allocation or resource change
* @param capability The target resource capability of the container
*/
- public abstract void requestContainerResourceChange(
- Container container, Resource capability);
+ @Deprecated
+ public void requestContainerResourceChange(
+ Container container, Resource capability) {
+ Preconditions.checkNotNull(container, "Container cannot be null!!");
+ Preconditions.checkNotNull(capability,
+ "UpdateContainerRequest cannot be null!!");
+ requestContainerUpdate(container, UpdateContainerRequest.newInstance(
+ container.getVersion(), container.getId(),
+ Resources.fitsIn(capability, container.getResource()) ?
+ ContainerUpdateType.DECREASE_RESOURCE :
+ ContainerUpdateType.INCREASE_RESOURCE,
+ capability, null));
+ }
+
+ /**
+ * Request a container update before calling <code>allocate</code>.
+ * Any previous pending update request of the same container will be
+ * removed.
+ *
+ * @param container The container returned from the last successful resource
+ * allocation or update
+ * @param updateContainerRequest The <code>UpdateContainerRequest</code>.
+ */
+ public abstract void requestContainerUpdate(
+ Container container, UpdateContainerRequest updateContainerRequest);
/**
* Release containers assigned by the Resource Manager. If the app cannot use
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 3e72d3f..9e2c0e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -207,9 +208,9 @@ extends AMRMClientAsync<T> {
}
@Override
- public void requestContainerResourceChange(
- Container container, Resource capability) {
- client.requestContainerResourceChange(container, capability);
+ public void requestContainerUpdate(Container container,
+ UpdateContainerRequest updateContainerRequest) {
+ client.requestContainerUpdate(container, updateContainerRequest);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 44fc1e0..7da91de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -169,15 +169,16 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
protected Set<ContainerId> pendingRelease = new TreeSet<ContainerId>();
// change map holds container resource change requests between two allocate()
// calls, and are cleared after each successful allocate() call.
- protected final Map<ContainerId, SimpleEntry<Container, Resource>> change =
- new HashMap<>();
+ protected final Map<ContainerId,
+ SimpleEntry<Container, UpdateContainerRequest>> change = new HashMap<>();
// pendingChange map holds history of container resource change requests in
// case AM needs to reregister with the ResourceManager.
// Change requests are removed from this map if RM confirms the change
// through allocate response, or if RM confirms that the container has been
// completed.
- protected final Map<ContainerId, SimpleEntry<Container, Resource>>
- pendingChange = new HashMap<>();
+ protected final Map<ContainerId,
+ SimpleEntry<Container, UpdateContainerRequest>> pendingChange =
+ new HashMap<>();
public AMRMClientImpl() {
super(AMRMClientImpl.class.getName());
@@ -259,7 +260,7 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
AllocateRequest allocateRequest = null;
List<String> blacklistToAdd = new ArrayList<String>();
List<String> blacklistToRemove = new ArrayList<String>();
- Map<ContainerId, SimpleEntry<Container, Resource>> oldChange =
+ Map<ContainerId, SimpleEntry<Container, UpdateContainerRequest>> oldChange =
new HashMap<>();
try {
synchronized (this) {
@@ -374,14 +375,15 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
//
// Only insert entries from the cached oldChange map
// that do not exist in the current change map:
- for (Map.Entry<ContainerId, SimpleEntry<Container, Resource>> entry :
+ for (Map.Entry<ContainerId,
+ SimpleEntry<Container, UpdateContainerRequest>> entry :
oldChange.entrySet()) {
ContainerId oldContainerId = entry.getKey();
Container oldContainer = entry.getValue().getKey();
- Resource oldResource = entry.getValue().getValue();
+ UpdateContainerRequest oldupdate = entry.getValue().getValue();
if (change.get(oldContainerId) == null) {
change.put(
- oldContainerId, new SimpleEntry<>(oldContainer, oldResource));
+ oldContainerId, new SimpleEntry<>(oldContainer, oldupdate));
}
}
blacklistAdditions.addAll(blacklistToAdd);
@@ -394,19 +396,17 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
private List<UpdateContainerRequest> createUpdateList() {
List<UpdateContainerRequest> updateList = new ArrayList<>();
- for (Map.Entry<ContainerId, SimpleEntry<Container, Resource>> entry :
- change.entrySet()) {
- Resource targetCapability = entry.getValue().getValue();
- Resource currCapability = entry.getValue().getKey().getResource();
- int version = entry.getValue().getKey().getVersion();
+ for (Map.Entry<ContainerId, SimpleEntry<Container,
+ UpdateContainerRequest>> entry : change.entrySet()) {
+ Resource targetCapability = entry.getValue().getValue().getCapability();
+ ExecutionType targetExecType =
+ entry.getValue().getValue().getExecutionType();
ContainerUpdateType updateType =
- ContainerUpdateType.INCREASE_RESOURCE;
- if (Resources.fitsIn(targetCapability, currCapability)) {
- updateType = ContainerUpdateType.DECREASE_RESOURCE;
- }
+ entry.getValue().getValue().getContainerUpdateType();
+ int version = entry.getValue().getKey().getVersion();
updateList.add(
UpdateContainerRequest.newInstance(version, entry.getKey(),
- updateType, targetCapability, null));
+ updateType, targetCapability, targetExecType));
}
return updateList;
}
@@ -591,21 +591,47 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
@Override
- public synchronized void requestContainerResourceChange(
- Container container, Resource capability) {
- validateContainerResourceChangeRequest(
- container.getId(), container.getResource(), capability);
+ public synchronized void requestContainerUpdate(
+ Container container, UpdateContainerRequest updateContainerRequest) {
+ Preconditions.checkNotNull(container, "Container cannot be null!!");
+ Preconditions.checkNotNull(updateContainerRequest,
+ "UpdateContainerRequest cannot be null!!");
+ LOG.info("Requesting Container update : " +
+ "container=" + container + ", " +
+ "updateType=" + updateContainerRequest.getContainerUpdateType() + ", " +
+ "targetCapability=" + updateContainerRequest.getCapability() + ", " +
+ "targetExecType=" + updateContainerRequest.getExecutionType());
+ if (updateContainerRequest.getCapability() != null &&
+ updateContainerRequest.getExecutionType() == null) {
+ validateContainerResourceChangeRequest(
+ updateContainerRequest.getContainerUpdateType(),
+ container.getId(), container.getResource(),
+ updateContainerRequest.getCapability());
+ } else if (updateContainerRequest.getExecutionType() != null &&
+ updateContainerRequest.getCapability() == null) {
+ validateContainerExecTypeChangeRequest(
+ updateContainerRequest.getContainerUpdateType(),
+ container.getId(), container.getExecutionType(),
+ updateContainerRequest.getExecutionType());
+ } else if (updateContainerRequest.getExecutionType() == null &&
+ updateContainerRequest.getCapability() == null) {
+ throw new IllegalArgumentException("Both target Capability and" +
+ "target Execution Type are null");
+ } else {
+ throw new IllegalArgumentException("Support currently exists only for" +
+ " EITHER update of Capability OR update of Execution Type NOT both");
+ }
if (change.get(container.getId()) == null) {
change.put(container.getId(),
- new SimpleEntry<>(container, capability));
+ new SimpleEntry<>(container, updateContainerRequest));
} else {
- change.get(container.getId()).setValue(capability);
+ change.get(container.getId()).setValue(updateContainerRequest);
}
if (pendingChange.get(container.getId()) == null) {
pendingChange.put(container.getId(),
- new SimpleEntry<>(container, capability));
+ new SimpleEntry<>(container, updateContainerRequest));
} else {
- pendingChange.get(container.getId()).setValue(capability);
+ pendingChange.get(container.getId()).setValue(updateContainerRequest);
}
}
@@ -755,7 +781,8 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
private void validateContainerResourceChangeRequest(
- ContainerId containerId, Resource original, Resource target) {
+ ContainerUpdateType updateType, ContainerId containerId,
+ Resource original, Resource target) {
Preconditions.checkArgument(containerId != null,
"ContainerId cannot be null");
Preconditions.checkArgument(original != null,
@@ -768,6 +795,36 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
Preconditions.checkArgument(!Resources.equals(Resources.none(), target)
&& Resources.fitsIn(Resources.none(), target),
"Target resource capability must be greater than 0");
+ if (ContainerUpdateType.DECREASE_RESOURCE == updateType) {
+ Preconditions.checkArgument(Resources.fitsIn(target, original),
+ "Target resource capability must fit in Original capability");
+ } else {
+ Preconditions.checkArgument(Resources.fitsIn(original, target),
+ "Target resource capability must be more than Original capability");
+
+ }
+ }
+
+ private void validateContainerExecTypeChangeRequest(
+ ContainerUpdateType updateType, ContainerId containerId,
+ ExecutionType original, ExecutionType target) {
+ Preconditions.checkArgument(containerId != null,
+ "ContainerId cannot be null");
+ Preconditions.checkArgument(original != null,
+ "Original Execution Type cannot be null");
+ Preconditions.checkArgument(target != null,
+ "Target Execution Type cannot be null");
+ if (ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateType) {
+ Preconditions.checkArgument(target == ExecutionType.OPPORTUNISTIC
+ && original == ExecutionType.GUARANTEED,
+ "Incorrect Container update request, target should be" +
+ " OPPORTUNISTIC and original should be GUARANTEED");
+ } else {
+ Preconditions.checkArgument(target == ExecutionType.GUARANTEED
+ && original == ExecutionType.OPPORTUNISTIC,
+ "Incorrect Container update request, target should be" +
+ " GUARANTEED and original should be OPPORTUNISTIC");
+ }
}
private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 8b1bbc7..4f73bac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -51,29 +51,7 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NMToken;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -1058,26 +1036,36 @@ public class TestAMRMClient {
(AMRMClientImpl<ContainerRequest>) amClient;
Assert.assertEquals(0, amClientImpl.change.size());
// verify newer request overwrites older request for the container1
- amClientImpl.requestContainerResourceChange(
- container1, Resource.newInstance(2048, 1));
- amClientImpl.requestContainerResourceChange(
- container1, Resource.newInstance(4096, 1));
+ amClientImpl.requestContainerUpdate(container1,
+ UpdateContainerRequest.newInstance(container1.getVersion(),
+ container1.getId(), ContainerUpdateType.INCREASE_RESOURCE,
+ Resource.newInstance(2048, 1), null));
+ amClientImpl.requestContainerUpdate(container1,
+ UpdateContainerRequest.newInstance(container1.getVersion(),
+ container1.getId(), ContainerUpdateType.INCREASE_RESOURCE,
+ Resource.newInstance(4096, 1), null));
Assert.assertEquals(Resource.newInstance(4096, 1),
- amClientImpl.change.get(container1.getId()).getValue());
+ amClientImpl.change.get(container1.getId()).getValue().getCapability());
// verify new decrease request cancels old increase request for container1
- amClientImpl.requestContainerResourceChange(
- container1, Resource.newInstance(512, 1));
+ amClientImpl.requestContainerUpdate(container1,
+ UpdateContainerRequest.newInstance(container1.getVersion(),
+ container1.getId(), ContainerUpdateType.DECREASE_RESOURCE,
+ Resource.newInstance(512, 1), null));
Assert.assertEquals(Resource.newInstance(512, 1),
- amClientImpl.change.get(container1.getId()).getValue());
+ amClientImpl.change.get(container1.getId()).getValue().getCapability());
// request resource increase for container2
- amClientImpl.requestContainerResourceChange(
- container2, Resource.newInstance(2048, 1));
+ amClientImpl.requestContainerUpdate(container2,
+ UpdateContainerRequest.newInstance(container2.getVersion(),
+ container2.getId(), ContainerUpdateType.INCREASE_RESOURCE,
+ Resource.newInstance(2048, 1), null));
Assert.assertEquals(Resource.newInstance(2048, 1),
- amClientImpl.change.get(container2.getId()).getValue());
+ amClientImpl.change.get(container2.getId()).getValue().getCapability());
// verify release request will cancel pending change requests for the same
// container
- amClientImpl.requestContainerResourceChange(
- container3, Resource.newInstance(2048, 1));
+ amClientImpl.requestContainerUpdate(container3,
+ UpdateContainerRequest.newInstance(container3.getVersion(),
+ container3.getId(), ContainerUpdateType.INCREASE_RESOURCE,
+ Resource.newInstance(2048, 1), null));
Assert.assertEquals(3, amClientImpl.pendingChange.size());
amClientImpl.releaseAssignedContainer(container3.getId());
Assert.assertEquals(2, amClientImpl.pendingChange.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index ac77446..39a7633 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -233,8 +234,11 @@ public class TestAMRMClientOnRMRestart {
nm1.nodeHeartbeat(containerId.getApplicationAttemptId(),
containerId.getContainerId(), ContainerState.RUNNING);
dispatcher.await();
- amClient.requestContainerResourceChange(
- container, Resource.newInstance(2048, 1));
+ amClient.requestContainerUpdate(
+ container, UpdateContainerRequest.newInstance(
+ container.getVersion(), container.getId(),
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resource.newInstance(2048, 1), null));
it.remove();
allocateResponse = amClient.allocate(0.3f);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
index 802c207..305d18b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -44,6 +45,8 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -54,6 +57,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -66,13 +72,17 @@ import org.junit.Test;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
/**
* Class that tests the allocation of OPPORTUNISTIC containers through the
@@ -83,7 +93,6 @@ public class TestOpportunisticContainerAllocation {
private static MiniYARNCluster yarnCluster = null;
private static YarnClient yarnClient = null;
private static List<NodeReport> nodeReports = null;
- private static ApplicationAttemptId attemptId = null;
private static int nodeCount = 3;
private static final int ROLLING_INTERVAL_SEC = 13;
@@ -92,12 +101,22 @@ public class TestOpportunisticContainerAllocation {
private static Resource capability;
private static Priority priority;
private static Priority priority2;
+ private static Priority priority3;
+ private static Priority priority4;
private static String node;
private static String rack;
private static String[] nodes;
private static String[] racks;
private final static int DEFAULT_ITERATION = 3;
+ // Per test..
+ private ApplicationAttemptId attemptId = null;
+ private AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
+ private long availMB;
+ private int availVCores;
+ private long allocMB;
+ private int allocVCores;
+
@BeforeClass
public static void setup() throws Exception {
// start minicluster
@@ -106,7 +125,7 @@ public class TestOpportunisticContainerAllocation {
YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
ROLLING_INTERVAL_SEC);
conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, AM_EXPIRE_MS);
- conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
+ conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1000);
// set the minimum allocation so that resource decrease can go under 1024
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
conf.setBoolean(
@@ -129,7 +148,9 @@ public class TestOpportunisticContainerAllocation {
priority = Priority.newInstance(1);
priority2 = Priority.newInstance(2);
- capability = Resource.newInstance(1024, 1);
+ priority3 = Priority.newInstance(3);
+ priority4 = Priority.newInstance(4);
+ capability = Resource.newInstance(512, 1);
node = nodeReports.get(0).getNodeId().getHost();
rack = nodeReports.get(0).getRackName();
@@ -193,10 +214,35 @@ public class TestOpportunisticContainerAllocation {
UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
appAttempt.getAMRMToken()
.setService(ClientRMProxy.getAMRMTokenService(conf));
+
+ // start am rm client
+ amClient = (AMRMClientImpl<AMRMClient.ContainerRequest>)AMRMClient
+ .createAMRMClient();
+
+ //setting an instance NMTokenCache
+ amClient.setNMTokenCache(new NMTokenCache());
+ //asserting we are not using the singleton instance cache
+ Assert.assertNotSame(NMTokenCache.getSingleton(),
+ amClient.getNMTokenCache());
+
+ amClient.init(conf);
+ amClient.start();
+
+ amClient.registerApplicationMaster("Host", 10000, "");
}
@After
public void cancelApp() throws YarnException, IOException {
+ try {
+ amClient
+ .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null,
+ null);
+ } finally {
+ if (amClient != null &&
+ amClient.getServiceState() == Service.STATE.STARTED) {
+ amClient.stop();
+ }
+ }
yarnClient.killApplication(attemptId.getApplicationId());
attemptId = null;
}
@@ -214,43 +260,254 @@ public class TestOpportunisticContainerAllocation {
}
@Test(timeout = 60000)
- public void testAMRMClient() throws YarnException, IOException {
- AMRMClient<AMRMClient.ContainerRequest> amClient = null;
+ public void testPromotionFromAcquired() throws YarnException, IOException {
+ // setup container request
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+
+ int oppContainersRequestedAny =
+ amClient.getTable(0).get(priority2, ResourceRequest.ANY,
+ ExecutionType.OPPORTUNISTIC, capability).remoteRequest
+ .getNumContainers();
+
+ assertEquals(1, oppContainersRequestedAny);
+
+ assertEquals(1, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ // RM should allocate container within 2 calls to allocate()
+ int allocatedContainerCount = 0;
+ Map<ContainerId, Container> allocatedOpportContainers = new HashMap<>();
+ int iterationsLeft = 50;
+
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0,
+ amClient.getNMTokenCache().numberOfTokensInCache());
+ HashMap<String, Token> receivedNMTokens = new HashMap<>();
+
+ updateMetrics("Before Opp Allocation");
+
+ while (allocatedContainerCount < oppContainersRequestedAny
+ && iterationsLeft-- > 0) {
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ allocatedContainerCount +=
+ allocResponse.getAllocatedContainers().size();
+ for (Container container : allocResponse.getAllocatedContainers()) {
+ if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
+ allocatedOpportContainers.put(container.getId(), container);
+ removeCR(container);
+ }
+ }
+
+ for (NMToken token : allocResponse.getNMTokens()) {
+ String nodeID = token.getNodeId().toString();
+ receivedNMTokens.put(nodeID, token.getToken());
+ }
+
+ if (allocatedContainerCount < oppContainersRequestedAny) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+
+ assertEquals(oppContainersRequestedAny, allocatedContainerCount);
+ assertEquals(oppContainersRequestedAny, allocatedOpportContainers.size());
+
+ updateMetrics("After Opp Allocation / Before Promotion");
+
try {
- // start am rm client
- amClient = AMRMClient.<AMRMClient.ContainerRequest>createAMRMClient();
+ Container c = allocatedOpportContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.OPPORTUNISTIC));
+ Assert.fail("Should throw Exception..");
+ } catch (IllegalArgumentException e) {
+ System.out.println("## " + e.getMessage());
+ Assert.assertTrue(e.getMessage().contains(
+ "target should be GUARANTEED and original should be OPPORTUNISTIC"));
+ }
- //setting an instance NMTokenCache
- amClient.setNMTokenCache(new NMTokenCache());
- //asserting we are not using the singleton instance cache
- Assert.assertNotSame(NMTokenCache.getSingleton(),
- amClient.getNMTokenCache());
+ Container c = allocatedOpportContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED));
+ iterationsLeft = 120;
+ Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
+ // do a few iterations to ensure RM is not going to send new containers
+ while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
+ // inform RM of rejection
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ // RM did not send new containers because AM does not need any
+ if (allocResponse.getUpdatedContainers() != null) {
+ for (UpdatedContainer updatedContainer : allocResponse
+ .getUpdatedContainers()) {
+ System.out.println("Got update..");
+ updatedContainers.put(updatedContainer.getContainer().getId(),
+ updatedContainer);
+ }
+ }
+ if (iterationsLeft > 0) {
+ // sleep to make sure NM's heartbeat
+ sleep(100);
+ }
+ }
- amClient.init(conf);
- amClient.start();
+ updateMetrics("After Promotion");
+
+ assertEquals(1, updatedContainers.size());
+ for (ContainerId cId : allocatedOpportContainers.keySet()) {
+ Container orig = allocatedOpportContainers.get(cId);
+ UpdatedContainer updatedContainer = updatedContainers.get(cId);
+ assertNotNull(updatedContainer);
+ assertEquals(ExecutionType.GUARANTEED,
+ updatedContainer.getContainer().getExecutionType());
+ assertEquals(orig.getResource(),
+ updatedContainer.getContainer().getResource());
+ assertEquals(orig.getNodeId(),
+ updatedContainer.getContainer().getNodeId());
+ assertEquals(orig.getVersion() + 1,
+ updatedContainer.getContainer().getVersion());
+ }
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+ amClient.ask.clear();
+ }
- amClient.registerApplicationMaster("Host", 10000, "");
+ @Test(timeout = 60000)
+ public void testDemotionFromAcquired() throws YarnException, IOException {
+ // setup container request
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
- testOpportunisticAllocation(
- (AMRMClientImpl<AMRMClient.ContainerRequest>) amClient);
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority3));
- testAllocation((AMRMClientImpl<AMRMClient.ContainerRequest>)amClient);
+ int guarContainersRequestedAny = amClient.getTable(0).get(priority3,
+ ResourceRequest.ANY, ExecutionType.GUARANTEED, capability)
+ .remoteRequest.getNumContainers();
- amClient
- .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null,
- null);
+ assertEquals(1, guarContainersRequestedAny);
- } finally {
- if (amClient != null &&
- amClient.getServiceState() == Service.STATE.STARTED) {
- amClient.stop();
+ assertEquals(1, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ // RM should allocate container within 2 calls to allocate()
+ int allocatedContainerCount = 0;
+ Map<ContainerId, Container> allocatedGuarContainers = new HashMap<>();
+ int iterationsLeft = 50;
+
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0,
+ amClient.getNMTokenCache().numberOfTokensInCache());
+ HashMap<String, Token> receivedNMTokens = new HashMap<>();
+
+ updateMetrics("Before Guar Allocation");
+
+ while (allocatedContainerCount < guarContainersRequestedAny
+ && iterationsLeft-- > 0) {
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ allocatedContainerCount +=
+ allocResponse.getAllocatedContainers().size();
+ for (Container container : allocResponse.getAllocatedContainers()) {
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ allocatedGuarContainers.put(container.getId(), container);
+ removeCR(container);
+ }
+ }
+
+ for (NMToken token : allocResponse.getNMTokens()) {
+ String nodeID = token.getNodeId().toString();
+ receivedNMTokens.put(nodeID, token.getToken());
+ }
+
+ if (allocatedContainerCount < guarContainersRequestedAny) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+
+ assertEquals(guarContainersRequestedAny, allocatedContainerCount);
+ assertEquals(guarContainersRequestedAny, allocatedGuarContainers.size());
+
+ updateMetrics("After Guar Allocation / Before Demotion");
+
+ try {
+ Container c = allocatedGuarContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED));
+ Assert.fail("Should throw Exception..");
+ } catch (IllegalArgumentException e) {
+ System.out.println("## " + e.getMessage());
+ Assert.assertTrue(e.getMessage().contains(
+ "target should be OPPORTUNISTIC and original should be GUARANTEED"));
+ }
+
+ Container c = allocatedGuarContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
+ null, ExecutionType.OPPORTUNISTIC));
+ iterationsLeft = 120;
+ Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
+ // do a few iterations to ensure RM is not going to send new containers
+ while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
+ // inform RM of rejection
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ // RM did not send new containers because AM does not need any
+ if (allocResponse.getUpdatedContainers() != null) {
+ for (UpdatedContainer updatedContainer : allocResponse
+ .getUpdatedContainers()) {
+ System.out.println("Got update..");
+ updatedContainers.put(updatedContainer.getContainer().getId(),
+ updatedContainer);
+ }
+ }
+ if (iterationsLeft > 0) {
+ // sleep to make sure NM's heartbeat
+ sleep(100);
}
}
+
+ updateMetrics("After Demotion");
+
+ assertEquals(1, updatedContainers.size());
+ for (ContainerId cId : allocatedGuarContainers.keySet()) {
+ Container orig = allocatedGuarContainers.get(cId);
+ UpdatedContainer updatedContainer = updatedContainers.get(cId);
+ assertNotNull(updatedContainer);
+ assertEquals(ExecutionType.OPPORTUNISTIC,
+ updatedContainer.getContainer().getExecutionType());
+ assertEquals(orig.getResource(),
+ updatedContainer.getContainer().getResource());
+ assertEquals(orig.getNodeId(),
+ updatedContainer.getContainer().getNodeId());
+ assertEquals(orig.getVersion() + 1,
+ updatedContainer.getContainer().getVersion());
+ }
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+ amClient.ask.clear();
}
- private void testAllocation(
- final AMRMClientImpl<AMRMClient.ContainerRequest> amClient)
- throws YarnException, IOException {
+ @Test(timeout = 60000)
+ public void testMixedAllocationAndRelease() throws YarnException,
+ IOException {
// setup container request
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
@@ -274,6 +531,28 @@ public class TestOpportunisticContainerAllocation {
ExecutionTypeRequest.newInstance(
ExecutionType.OPPORTUNISTIC, true)));
+ int containersRequestedNode = amClient.getTable(0).get(priority,
+ node, ExecutionType.GUARANTEED, capability).remoteRequest
+ .getNumContainers();
+ int containersRequestedRack = amClient.getTable(0).get(priority,
+ rack, ExecutionType.GUARANTEED, capability).remoteRequest
+ .getNumContainers();
+ int containersRequestedAny = amClient.getTable(0).get(priority,
+ ResourceRequest.ANY, ExecutionType.GUARANTEED, capability)
+ .remoteRequest.getNumContainers();
+ int oppContainersRequestedAny =
+ amClient.getTable(0).get(priority2, ResourceRequest.ANY,
+ ExecutionType.OPPORTUNISTIC, capability).remoteRequest
+ .getNumContainers();
+
+ assertEquals(4, containersRequestedNode);
+ assertEquals(4, containersRequestedRack);
+ assertEquals(4, containersRequestedAny);
+ assertEquals(2, oppContainersRequestedAny);
+
+ assertEquals(4, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
amClient.removeContainerRequest(
new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
amClient.removeContainerRequest(
@@ -284,16 +563,16 @@ public class TestOpportunisticContainerAllocation {
ExecutionTypeRequest.newInstance(
ExecutionType.OPPORTUNISTIC, true)));
- int containersRequestedNode = amClient.getTable(0).get(priority,
+ containersRequestedNode = amClient.getTable(0).get(priority,
node, ExecutionType.GUARANTEED, capability).remoteRequest
.getNumContainers();
- int containersRequestedRack = amClient.getTable(0).get(priority,
+ containersRequestedRack = amClient.getTable(0).get(priority,
rack, ExecutionType.GUARANTEED, capability).remoteRequest
.getNumContainers();
- int containersRequestedAny = amClient.getTable(0).get(priority,
+ containersRequestedAny = amClient.getTable(0).get(priority,
ResourceRequest.ANY, ExecutionType.GUARANTEED, capability)
.remoteRequest.getNumContainers();
- int oppContainersRequestedAny =
+ oppContainersRequestedAny =
amClient.getTable(0).get(priority2, ResourceRequest.ANY,
ExecutionType.OPPORTUNISTIC, capability).remoteRequest
.getNumContainers();
@@ -309,7 +588,7 @@ public class TestOpportunisticContainerAllocation {
// RM should allocate container within 2 calls to allocate()
int allocatedContainerCount = 0;
int allocatedOpportContainerCount = 0;
- int iterationsLeft = 10;
+ int iterationsLeft = 50;
Set<ContainerId> releases = new TreeSet<>();
amClient.getNMTokenCache().clearCache();
@@ -324,8 +603,8 @@ public class TestOpportunisticContainerAllocation {
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
- allocatedContainerCount += allocResponse.getAllocatedContainers()
- .size();
+ allocatedContainerCount +=
+ allocResponse.getAllocatedContainers().size();
for (Container container : allocResponse.getAllocatedContainers()) {
if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
allocatedOpportContainerCount++;
@@ -345,9 +624,9 @@ public class TestOpportunisticContainerAllocation {
}
}
- assertEquals(allocatedContainerCount,
- containersRequestedAny + oppContainersRequestedAny);
- assertEquals(allocatedOpportContainerCount, oppContainersRequestedAny);
+ assertEquals(containersRequestedAny + oppContainersRequestedAny,
+ allocatedContainerCount);
+ assertEquals(oppContainersRequestedAny, allocatedOpportContainerCount);
for (ContainerId rejectContainerId : releases) {
amClient.releaseAssignedContainer(rejectContainerId);
}
@@ -395,26 +674,25 @@ public class TestOpportunisticContainerAllocation {
/**
* Tests allocation with requests comprising only opportunistic containers.
*/
- private void testOpportunisticAllocation(
- final AMRMClientImpl<AMRMClient.ContainerRequest> amClient)
- throws YarnException, IOException {
+ @Test(timeout = 60000)
+ public void testOpportunisticAllocation() throws YarnException, IOException {
// setup container request
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority, 0,
+ new AMRMClient.ContainerRequest(capability, null, null, priority3, 0,
true, null,
ExecutionTypeRequest.newInstance(
ExecutionType.OPPORTUNISTIC, true)));
amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority, 0,
+ new AMRMClient.ContainerRequest(capability, null, null, priority3, 0,
true, null,
ExecutionTypeRequest.newInstance(
ExecutionType.OPPORTUNISTIC, true)));
int oppContainersRequestedAny =
- amClient.getTable(0).get(priority, ResourceRequest.ANY,
+ amClient.getTable(0).get(priority3, ResourceRequest.ANY,
ExecutionType.OPPORTUNISTIC, capability).remoteRequest
.getNumContainers();
@@ -456,9 +734,43 @@ public class TestOpportunisticContainerAllocation {
}
}
+ assertEquals(oppContainersRequestedAny, allocatedContainerCount);
assertEquals(1, receivedNMTokens.values().size());
}
+ private void removeCR(Container container) {
+ List<? extends Collection<AMRMClient.ContainerRequest>>
+ matchingRequests = amClient.getMatchingRequests(container
+ .getPriority(),
+ ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC,
+ container.getResource());
+ Set<AMRMClient.ContainerRequest> toRemove = new HashSet<>();
+ for (Collection<AMRMClient.ContainerRequest> rc : matchingRequests) {
+ for (AMRMClient.ContainerRequest cr : rc) {
+ toRemove.add(cr);
+ }
+ }
+ for (AMRMClient.ContainerRequest cr : toRemove) {
+ amClient.removeContainerRequest(cr);
+ }
+ }
+
+ private void updateMetrics(String msg) {
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler)yarnCluster.getResourceManager()
+ .getResourceScheduler();
+ availMB = scheduler.getRootQueueMetrics().getAvailableMB();
+ availVCores = scheduler.getRootQueueMetrics().getAvailableVirtualCores();
+ allocMB = scheduler.getRootQueueMetrics().getAllocatedMB();
+ allocVCores = scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
+ System.out.println("## METRICS (" + msg + ")==>");
+ System.out.println(" : availMB=" + availMB + ", " +
+ "availVCores=" +availVCores + ", " +
+ "allocMB=" + allocMB + ", " +
+ "allocVCores=" + allocVCores + ", ");
+ System.out.println("<== ##");
+ }
+
private void sleep(int sleepTime) {
try {
Thread.sleep(sleepTime);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UpdateContainerErrorPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UpdateContainerErrorPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UpdateContainerErrorPBImpl.java
index fb6c1a7..8ff9d9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UpdateContainerErrorPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/UpdateContainerErrorPBImpl.java
@@ -74,6 +74,22 @@ public class UpdateContainerErrorPBImpl extends UpdateContainerError {
}
@Override
+ public int getCurrentContainerVersion() {
+ YarnServiceProtos.UpdateContainerErrorProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasCurrentContainerVersion()) {
+ return 0;
+ }
+ return p.getCurrentContainerVersion();
+ }
+
+ @Override
+ public void setCurrentContainerVersion(int containerVersion) {
+ maybeInitBuilder();
+ builder.setCurrentContainerVersion(containerVersion);
+ }
+
+ @Override
public UpdateContainerRequest getUpdateContainerRequest() {
YarnServiceProtos.UpdateContainerErrorProtoOrBuilder p = viaProto ? proto
: builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 94bfd58..224a1da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -188,19 +188,25 @@ public class RMServerUtils {
}
}
}
- checkAndcreateUpdateError(updateErrors, updateReq, msg);
+ checkAndcreateUpdateError(updateErrors, updateReq, rmContainer, msg);
}
return updateRequests;
}
private static void checkAndcreateUpdateError(
List<UpdateContainerError> errors, UpdateContainerRequest updateReq,
- String msg) {
+ RMContainer rmContainer, String msg) {
if (msg != null) {
UpdateContainerError updateError = RECORD_FACTORY
.newRecordInstance(UpdateContainerError.class);
updateError.setReason(msg);
updateError.setUpdateContainerRequest(updateReq);
+ if (rmContainer != null) {
+ updateError.setCurrentContainerVersion(
+ rmContainer.getContainer().getVersion());
+ } else {
+ updateError.setCurrentContainerVersion(-1);
+ }
errors.add(updateError);
}
}
@@ -216,9 +222,7 @@ public class RMServerUtils {
// version
if (msg == null && updateReq.getContainerVersion() !=
rmContainer.getContainer().getVersion()) {
- msg = INCORRECT_CONTAINER_VERSION_ERROR + "|"
- + updateReq.getContainerVersion() + "|"
- + rmContainer.getContainer().getVersion();
+ msg = INCORRECT_CONTAINER_VERSION_ERROR;
}
// No more than 1 container update per request.
if (msg == null &&
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 641ef64..b083642 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -251,8 +251,11 @@ public class TestOpportunisticContainerAllocatorAMService {
Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
Assert.assertEquals(1, allocateResponse.getUpdateErrors().size());
- Assert.assertEquals("INCORRECT_CONTAINER_VERSION_ERROR|1|0",
+ Assert.assertEquals("INCORRECT_CONTAINER_VERSION_ERROR",
allocateResponse.getUpdateErrors().get(0).getReason());
+ Assert.assertEquals(0,
+ allocateResponse.getUpdateErrors().get(0)
+ .getCurrentContainerVersion());
Assert.assertEquals(container.getId(),
allocateResponse.getUpdateErrors().get(0)
.getUpdateContainerRequest().getContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf106fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
index c5829cf..74cecf2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
@@ -275,8 +275,10 @@ public class TestIncreaseAllocationExpirer {
Resources.createResource(5 * GB), null)));
List<UpdateContainerError> updateErrors = response.getUpdateErrors();
Assert.assertEquals(1, updateErrors.size());
- Assert.assertEquals("INCORRECT_CONTAINER_VERSION_ERROR|0|1",
+ Assert.assertEquals("INCORRECT_CONTAINER_VERSION_ERROR",
updateErrors.get(0).getReason());
+ Assert.assertEquals(1,
+ updateErrors.get(0).getCurrentContainerVersion());
// am1 asks to change containerId2 from 3GB to 5GB
am1.sendContainerResizingRequest(Collections.singletonList(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-3662. Federation Membership
State Store internal APIs.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
new file mode 100644
index 0000000..d4c5451
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterDeregisterRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterDeregisterRequestPBImpl
+ extends SubClusterDeregisterRequest {
+
+ private SubClusterDeregisterRequestProto proto =
+ SubClusterDeregisterRequestProto.getDefaultInstance();
+ private SubClusterDeregisterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterDeregisterRequestPBImpl() {
+ builder = SubClusterDeregisterRequestProto.newBuilder();
+ }
+
+ public SubClusterDeregisterRequestPBImpl(
+ SubClusterDeregisterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterDeregisterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterDeregisterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ SubClusterDeregisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getSubClusterId());
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ return;
+ }
+ builder.setSubClusterId(convertToProtoFormat(subClusterId));
+ }
+
+ @Override
+ public SubClusterState getState() {
+ SubClusterDeregisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getState());
+ }
+
+ @Override
+ public void setState(SubClusterState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearState();
+ return;
+ }
+ builder.setState(convertToProtoFormat(state));
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+ return new SubClusterIdPBImpl(sc);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+ return ((SubClusterIdPBImpl) sc).getProto();
+ }
+
+ private SubClusterState convertFromProtoFormat(SubClusterStateProto state) {
+ return SubClusterState.valueOf(state.name());
+ }
+
+ private SubClusterStateProto convertToProtoFormat(SubClusterState state) {
+ return SubClusterStateProto.valueOf(state.name());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java
new file mode 100644
index 0000000..9e00796
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterDeregisterResponse}.
+ */
+@Private
+@Unstable
+public class SubClusterDeregisterResponsePBImpl
+ extends SubClusterDeregisterResponse {
+
+ private SubClusterDeregisterResponseProto proto =
+ SubClusterDeregisterResponseProto.getDefaultInstance();
+ private SubClusterDeregisterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterDeregisterResponsePBImpl() {
+ builder = SubClusterDeregisterResponseProto.newBuilder();
+ }
+
+ public SubClusterDeregisterResponsePBImpl(
+ SubClusterDeregisterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterDeregisterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
new file mode 100644
index 0000000..ca6b154
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterHeartbeatRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterHeartbeatRequestPBImpl
+ extends SubClusterHeartbeatRequest {
+
+ private SubClusterHeartbeatRequestProto proto =
+ SubClusterHeartbeatRequestProto.getDefaultInstance();
+ private SubClusterHeartbeatRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterId subClusterId = null;
+
+ public SubClusterHeartbeatRequestPBImpl() {
+ builder = SubClusterHeartbeatRequestProto.newBuilder();
+ }
+
+ public SubClusterHeartbeatRequestPBImpl(
+ SubClusterHeartbeatRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterHeartbeatRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterHeartbeatRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterId != null) {
+ builder.setSubClusterId(convertToProtoFormat(this.subClusterId));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterId != null) {
+ return this.subClusterId;
+ }
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ this.subClusterId = convertFromProtoFormat(p.getSubClusterId());
+ return this.subClusterId;
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ }
+ this.subClusterId = subClusterId;
+ }
+
+ @Override
+ public long getLastHeartBeat() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getLastHeartBeat();
+ }
+
+ @Override
+ public void setLastHeartBeat(long time) {
+ maybeInitBuilder();
+ builder.setLastHeartBeat(time);
+ }
+
+ @Override
+ public SubClusterState getState() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getState());
+ }
+
+ @Override
+ public void setState(SubClusterState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearState();
+ return;
+ }
+ builder.setState(convertToProtoFormat(state));
+ }
+
+ @Override
+ public String getCapability() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasCapability()) ? p.getCapability() : null;
+ }
+
+ @Override
+ public void setCapability(String capability) {
+ maybeInitBuilder();
+ if (capability == null) {
+ builder.clearCapability();
+ return;
+ }
+ builder.setCapability(capability);
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto clusterId) {
+ return new SubClusterIdPBImpl(clusterId);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId clusterId) {
+ return ((SubClusterIdPBImpl) clusterId).getProto();
+ }
+
+ private SubClusterState convertFromProtoFormat(SubClusterStateProto state) {
+ return SubClusterState.valueOf(state.name());
+ }
+
+ private SubClusterStateProto convertToProtoFormat(SubClusterState state) {
+ return SubClusterStateProto.valueOf(state.name());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java
new file mode 100644
index 0000000..2020c1a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterHeartbeatResponse}.
+ */
+@Private
+@Unstable
+public class SubClusterHeartbeatResponsePBImpl
+ extends SubClusterHeartbeatResponse {
+
+ private SubClusterHeartbeatResponseProto proto =
+ SubClusterHeartbeatResponseProto.getDefaultInstance();
+ private SubClusterHeartbeatResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterHeartbeatResponsePBImpl() {
+ builder = SubClusterHeartbeatResponseProto.newBuilder();
+ }
+
+ public SubClusterHeartbeatResponsePBImpl(
+ SubClusterHeartbeatResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterHeartbeatResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java
new file mode 100644
index 0000000..1bf96bf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterId}.
+ */
+@Private
+@Unstable
+public class SubClusterIdPBImpl extends SubClusterId {
+
+ private SubClusterIdProto proto = SubClusterIdProto.getDefaultInstance();
+ private SubClusterIdProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterIdPBImpl() {
+ builder = SubClusterIdProto.newBuilder();
+ }
+
+ public SubClusterIdPBImpl(SubClusterIdProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterIdProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterIdProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public String getId() {
+ SubClusterIdProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getId();
+ }
+
+ @Override
+ protected void setId(String subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearId();
+ return;
+ }
+ builder.setId(subClusterId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
new file mode 100644
index 0000000..b650b5f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterInfo}.
+ */
+@Private
+@Unstable
+public class SubClusterInfoPBImpl extends SubClusterInfo {
+
+ private SubClusterInfoProto proto = SubClusterInfoProto.getDefaultInstance();
+ private SubClusterInfoProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterId subClusterId = null;
+
+ public SubClusterInfoPBImpl() {
+ builder = SubClusterInfoProto.newBuilder();
+ }
+
+ public SubClusterInfoPBImpl(SubClusterInfoProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterInfoProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterInfoProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterId != null) {
+ builder.setSubClusterId(convertToProtoFormat(this.subClusterId));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterId != null) {
+ return this.subClusterId;
+ }
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ this.subClusterId = convertFromProtoFormat(p.getSubClusterId());
+ return this.subClusterId;
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ }
+ this.subClusterId = subClusterId;
+ }
+
+ @Override
+ public String getAMRMServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasAMRMServiceAddress()) ? p.getAMRMServiceAddress() : null;
+ }
+
+ @Override
+ public void setAMRMServiceAddress(String amRMServiceAddress) {
+ maybeInitBuilder();
+ if (amRMServiceAddress == null) {
+ builder.clearAMRMServiceAddress();
+ return;
+ }
+ builder.setAMRMServiceAddress(amRMServiceAddress);
+ }
+
+ @Override
+ public String getClientRMServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasClientRMServiceAddress()) ? p.getClientRMServiceAddress()
+ : null;
+ }
+
+ @Override
+ public void setClientRMServiceAddress(String clientRMServiceAddress) {
+ maybeInitBuilder();
+ if (clientRMServiceAddress == null) {
+ builder.clearClientRMServiceAddress();
+ return;
+ }
+ builder.setClientRMServiceAddress(clientRMServiceAddress);
+ }
+
+ @Override
+ public String getRMAdminServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasRMAdminServiceAddress()) ? p.getRMAdminServiceAddress() : null;
+ }
+
+ @Override
+ public void setRMAdminServiceAddress(String rmAdminServiceAddress) {
+ maybeInitBuilder();
+ if (rmAdminServiceAddress == null) {
+ builder.clearRMAdminServiceAddress();
+ return;
+ }
+ builder.setRMAdminServiceAddress(rmAdminServiceAddress);
+ }
+
+ @Override
+ public String getRMWebServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasRMWebServiceAddress()) ? p.getRMWebServiceAddress() : null;
+ }
+
+ @Override
+ public void setRMWebServiceAddress(String rmWebServiceAddress) {
+ maybeInitBuilder();
+ if (rmWebServiceAddress == null) {
+ builder.clearRMWebServiceAddress();
+ return;
+ }
+ builder.setRMWebServiceAddress(rmWebServiceAddress);
+ }
+
+ @Override
+ public long getLastHeartBeat() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getLastHeartBeat();
+ }
+
+ @Override
+ public void setLastHeartBeat(long time) {
+ maybeInitBuilder();
+ builder.setLastHeartBeat(time);
+ }
+
+ @Override
+ public SubClusterState getState() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getState());
+ }
+
+ @Override
+ public void setState(SubClusterState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearState();
+ return;
+ }
+ builder.setState(convertToProtoFormat(state));
+ }
+
+ @Override
+ public long getLastStartTime() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasLastStartTime()) ? p.getLastStartTime() : 0;
+ }
+
+ @Override
+ public void setLastStartTime(long lastStartTime) {
+ Preconditions.checkNotNull(builder);
+ builder.setLastStartTime(lastStartTime);
+ }
+
+ @Override
+ public String getCapability() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasCapability()) ? p.getCapability() : null;
+ }
+
+ @Override
+ public void setCapability(String capability) {
+ maybeInitBuilder();
+ if (capability == null) {
+ builder.clearCapability();
+ return;
+ }
+ builder.setCapability(capability);
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto clusterId) {
+ return new SubClusterIdPBImpl(clusterId);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId clusterId) {
+ return ((SubClusterIdPBImpl) clusterId).getProto();
+ }
+
+ private SubClusterState convertFromProtoFormat(SubClusterStateProto state) {
+ return SubClusterState.valueOf(state.name());
+ }
+
+ private SubClusterStateProto convertToProtoFormat(SubClusterState state) {
+ return SubClusterStateProto.valueOf(state.name());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
new file mode 100644
index 0000000..3429cc9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterRegisterRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterRegisterRequestPBImpl extends SubClusterRegisterRequest {
+
+ private SubClusterRegisterRequestProto proto =
+ SubClusterRegisterRequestProto.getDefaultInstance();
+ private SubClusterRegisterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterInfo subClusterInfo = null;
+
+ public SubClusterRegisterRequestPBImpl() {
+ builder = SubClusterRegisterRequestProto.newBuilder();
+ }
+
+ public SubClusterRegisterRequestPBImpl(SubClusterRegisterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterRegisterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterRegisterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterInfo != null) {
+ builder.setSubClusterInfo(convertToProtoFormat(this.subClusterInfo));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterInfo getSubClusterInfo() {
+ SubClusterRegisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterInfo != null) {
+ return this.subClusterInfo;
+ }
+ if (!p.hasSubClusterInfo()) {
+ return null;
+ }
+ this.subClusterInfo = convertFromProtoFormat(p.getSubClusterInfo());
+ return this.subClusterInfo;
+ }
+
+ @Override
+ public void setSubClusterInfo(SubClusterInfo subClusterInfo) {
+ maybeInitBuilder();
+ if (subClusterInfo == null) {
+ builder.clearSubClusterInfo();
+ }
+ this.subClusterInfo = subClusterInfo;
+ }
+
+ private SubClusterInfo convertFromProtoFormat(
+ SubClusterInfoProto clusterInfo) {
+ return new SubClusterInfoPBImpl(clusterInfo);
+ }
+
+ private SubClusterInfoProto convertToProtoFormat(SubClusterInfo clusterInfo) {
+ return ((SubClusterInfoPBImpl) clusterInfo).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java
new file mode 100644
index 0000000..68930e3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterRegisterResponse}.
+ */
+@Private
+@Unstable
+public class SubClusterRegisterResponsePBImpl
+ extends SubClusterRegisterResponse {
+
+ private SubClusterRegisterResponseProto proto =
+ SubClusterRegisterResponseProto.getDefaultInstance();
+ private SubClusterRegisterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterRegisterResponsePBImpl() {
+ builder = SubClusterRegisterResponseProto.newBuilder();
+ }
+
+ public SubClusterRegisterResponsePBImpl(
+ SubClusterRegisterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterRegisterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java
new file mode 100644
index 0000000..2f85c48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java
new file mode 100644
index 0000000..9a9b282
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
new file mode 100644
index 0000000..1b2e53e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.federation.proto";
+option java_outer_classname = "YarnServerFederationProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+import "yarn_protos.proto";
+import "yarn_server_common_protos.proto";
+
+message SubClusterIdProto {
+ optional string id = 1;
+}
+
+enum SubClusterStateProto {
+ SC_NEW = 1;
+ SC_RUNNING = 2;
+ SC_UNHEALTHY = 3;
+ SC_DECOMMISSIONING = 4;
+ SC_LOST = 5;
+ SC_UNREGISTERED = 6;
+ SC_DECOMMISSIONED = 7;
+}
+
+message SubClusterInfoProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+ optional string aMRM_service_address = 2;
+ optional string client_rM_service_address = 3;
+ optional string rM_admin_service_address = 4;
+ optional string rM_web_service_address = 5;
+ optional int64 lastHeartBeat = 6;
+ optional SubClusterStateProto state = 7;
+ optional int64 lastStartTime = 8;
+ optional string capability = 9;
+}
+
+message SubClusterRegisterRequestProto {
+ optional SubClusterInfoProto sub_cluster_info = 1;
+}
+
+message SubClusterRegisterResponseProto {
+}
+
+message SubClusterHeartbeatRequestProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+ optional int64 lastHeartBeat = 2;
+ optional SubClusterStateProto state = 3;
+ optional string capability = 4;
+}
+
+message SubClusterHeartbeatResponseProto {
+}
+
+message SubClusterDeregisterRequestProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+ optional SubClusterStateProto state = 2;
+}
+
+message SubClusterDeregisterResponseProto {
+}
+
+message GetSubClusterInfoRequestProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+}
+
+message GetSubClusterInfoResponseProto {
+ optional SubClusterInfoProto sub_cluster_info = 1;
+}
+
+message GetSubClustersInfoRequestProto {
+ optional bool filter_inactive_subclusters = 1 [default = true];
+}
+
+message GetSubClustersInfoResponseProto {
+ repeated SubClusterInfoProto sub_cluster_infos = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
new file mode 100644
index 0000000..681edb1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.yarn.api.BasePBImplRecordsTest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterHeartbeatRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterHeartbeatResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test class for federation protocol records.
+ */
+public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ generateByNewInstance(ApplicationId.class);
+ generateByNewInstance(Version.class);
+ generateByNewInstance(SubClusterId.class);
+ generateByNewInstance(SubClusterInfo.class);
+ }
+
+ @Test
+ public void testSubClusterId() throws Exception {
+ validatePBImplRecord(SubClusterIdPBImpl.class, SubClusterIdProto.class);
+ }
+
+ @Test
+ public void testSubClusterInfo() throws Exception {
+ validatePBImplRecord(SubClusterInfoPBImpl.class, SubClusterInfoProto.class);
+ }
+
+ @Test
+ public void testSubClusterRegisterRequest() throws Exception {
+ validatePBImplRecord(SubClusterRegisterRequestPBImpl.class,
+ SubClusterRegisterRequestProto.class);
+ }
+
+ @Test
+ public void testSubClusterRegisterResponse() throws Exception {
+ validatePBImplRecord(SubClusterRegisterResponsePBImpl.class,
+ SubClusterRegisterResponseProto.class);
+ }
+
+ @Test
+ public void testSubClusterDeregisterRequest() throws Exception {
+ validatePBImplRecord(SubClusterDeregisterRequestPBImpl.class,
+ SubClusterDeregisterRequestProto.class);
+ }
+
+ @Test
+ public void testSubClusterDeregisterResponse() throws Exception {
+ validatePBImplRecord(SubClusterDeregisterResponsePBImpl.class,
+ SubClusterDeregisterResponseProto.class);
+ }
+
+ @Test
+ public void testSubClusterHeartbeatRequest() throws Exception {
+ validatePBImplRecord(SubClusterHeartbeatRequestPBImpl.class,
+ SubClusterHeartbeatRequestProto.class);
+ }
+
+ @Test
+ public void testSubClusterHeartbeatResponse() throws Exception {
+ validatePBImplRecord(SubClusterHeartbeatResponsePBImpl.class,
+ SubClusterHeartbeatResponseProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterRequest() throws Exception {
+ validatePBImplRecord(GetSubClusterInfoRequestPBImpl.class,
+ GetSubClusterInfoRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterResponse() throws Exception {
+ validatePBImplRecord(GetSubClusterInfoResponsePBImpl.class,
+ GetSubClusterInfoResponseProto.class);
+ }
+
+ @Test
+ public void testGetSubClustersInfoRequest() throws Exception {
+ validatePBImplRecord(GetSubClustersInfoRequestPBImpl.class,
+ GetSubClustersInfoRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClustersInfoResponse() throws Exception {
+ validatePBImplRecord(GetSubClustersInfoResponsePBImpl.class,
+ GetSubClustersInfoResponseProto.class);
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-5391. PolicyManager to tie
together Router/AMRM Federation policies. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5391. PolicyManager to tie together Router/AMRM Federation policies. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/112b99b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/112b99b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/112b99b9
Branch: refs/heads/YARN-2915
Commit: 112b99b944186956cdfd36cf57c58a41cd8f94c9
Parents: ed1764c
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 1 19:54:18 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../policies/AbstractPolicyManager.java | 175 +++++++++++++++++++
.../FederationPolicyInitializationContext.java | 3 +-
.../policies/UniformBroadcastPolicyManager.java | 56 ++++++
.../policies/WeightedLocalityPolicyManager.java | 67 +++++++
.../records/SubClusterPolicyConfiguration.java | 13 ++
.../policies/BasePolicyManagerTest.java | 108 ++++++++++++
...ionPolicyInitializationContextValidator.java | 5 +-
.../TestUniformBroadcastPolicyManager.java | 40 +++++
.../TestWeightedLocalityPolicyManager.java | 79 +++++++++
.../utils/FederationPoliciesTestUtil.java | 2 +-
10 files changed, 545 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
new file mode 100644
index 0000000..e77f2e3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides basic implementation for common methods that multiple
+ * policies will need to implement.
+ */
+public abstract class AbstractPolicyManager implements
+ FederationPolicyManager {
+
+ private String queue;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class routerFederationPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class amrmProxyFederationPolicy;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AbstractPolicyManager.class);
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ if (amrmProxyFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The parameter "
+ + "amrmProxyFederationPolicy should be initialized in "
+ + this.getClass().getSimpleName() + " constructor.");
+ }
+
+ try {
+ return (FederationAMRMProxyPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, amrmProxyFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+
+ }
+
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationRouterPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+
+ public FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ //checks that sub-types properly initialize the types of policies
+ if (routerFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The policy "
+ + "type should be initialized in " + this.getClass().getSimpleName()
+ + " constructor.");
+ }
+
+ try {
+ return (FederationRouterPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, routerFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ @Override
+ public String getQueue() {
+ return queue;
+ }
+
+ @Override
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ /**
+ * Common functionality to instantiate a reinitialize a {@link
+ * ConfigurableFederationPolicy}.
+ */
+ private ConfigurableFederationPolicy internalPolicyGetter(
+ final FederationPolicyInitializationContext federationPolicyContext,
+ ConfigurableFederationPolicy oldInstance, Class policy)
+ throws FederationPolicyInitializationException {
+
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ if (oldInstance == null || !oldInstance.getClass().equals(policy)) {
+ try {
+ oldInstance = (ConfigurableFederationPolicy) policy.newInstance();
+ } catch (InstantiationException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (IllegalAccessException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ //copying the context to avoid side-effects
+ FederationPolicyInitializationContext modifiedContext =
+ updateContext(federationPolicyContext,
+ oldInstance.getClass().getCanonicalName());
+
+ oldInstance.reinitialize(modifiedContext);
+ return oldInstance;
+ }
+
+ /**
+ * This method is used to copy-on-write the context, that will be passed
+ * downstream to the router/amrmproxy policies.
+ */
+ private FederationPolicyInitializationContext updateContext(
+ FederationPolicyInitializationContext federationPolicyContext,
+ String type) {
+ // copying configuration and context to avoid modification of original
+ SubClusterPolicyConfiguration newConf = SubClusterPolicyConfiguration
+ .newInstance(federationPolicyContext
+ .getSubClusterPolicyConfiguration());
+ newConf.setType(type);
+
+ return new FederationPolicyInitializationContext(newConf,
+ federationPolicyContext.getFederationSubclusterResolver(),
+ federationPolicyContext.getFederationStateStoreFacade(),
+ federationPolicyContext.getHomeSubcluster());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
index 46dd6eb..4d29a41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -41,10 +41,11 @@ public class FederationPolicyInitializationContext {
public FederationPolicyInitializationContext(
SubClusterPolicyConfiguration policy, SubClusterResolver resolver,
- FederationStateStoreFacade storeFacade) {
+ FederationStateStoreFacade storeFacade, SubClusterId home) {
this.federationPolicyConfiguration = policy;
this.federationSubclusterResolver = resolver;
this.federationStateStoreFacade = storeFacade;
+ this.homeSubcluster = home;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..a01f8fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class represents a simple implementation of a {@code
+ * FederationPolicyManager}.
+ *
+ * It combines the basic policies: {@link UniformRandomRouterPolicy} and
+ * {@link BroadcastAMRMProxyPolicy}, which are designed to work together and
+ * "spread" the load among sub-clusters uniformly.
+ *
+ * This simple policy might impose heavy load on the RMs and return more
+ * containers than a job requested as all requests are (replicated and)
+ * broadcasted.
+ */
+public class UniformBroadcastPolicyManager
+ extends AbstractPolicyManager {
+
+ public UniformBroadcastPolicyManager() {
+ //this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = UniformRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = ByteBuffer.allocate(0);
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..f3c6673
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link WeightedRandomRouterPolicy} for the router and a {@link
+ * LocalityMulticastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class WeightedLocalityPolicyManager
+ extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public WeightedLocalityPolicyManager() {
+ //this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = WeightedRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(
+ WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
index 2839139..52807d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -40,6 +40,7 @@ import java.nio.ByteBuffer;
@Unstable
public abstract class SubClusterPolicyConfiguration {
+
@Private
@Unstable
public static SubClusterPolicyConfiguration newInstance(String queue,
@@ -52,6 +53,18 @@ public abstract class SubClusterPolicyConfiguration {
return policy;
}
+ @Private
+ @Unstable
+ public static SubClusterPolicyConfiguration newInstance(
+ SubClusterPolicyConfiguration conf) {
+ SubClusterPolicyConfiguration policy =
+ Records.newRecord(SubClusterPolicyConfiguration.class);
+ policy.setQueue(conf.getQueue());
+ policy.setType(conf.getType());
+ policy.setParams(conf.getParams());
+ return policy;
+ }
+
/**
* Get the name of the queue for which we are configuring a policy.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
new file mode 100644
index 0000000..c609886
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class provides common test methods for testing {@code
+ * FederationPolicyManager}s.
+ */
+public abstract class BasePolicyManagerTest {
+
+
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected FederationPolicyManager wfp = null;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedPolicyManager;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedAMRMProxyPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedRouterPolicy;
+
+
+ @Test
+ public void testSerializeAndInstantiate() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy,
+ expectedRouterPolicy);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testSerializeAndInstantiateBad1() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, String.class,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad2() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ String.class, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad3() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, String.class);
+ }
+
+ protected static void serializeAndDeserializePolicyManager(
+ FederationPolicyManager wfp, Class policyManagerType,
+ Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception {
+
+ // serializeConf it in a context
+ SubClusterPolicyConfiguration fpc =
+ wfp.serializeConf();
+ fpc.setType(policyManagerType.getCanonicalName());
+ FederationPolicyInitializationContext context = new
+ FederationPolicyInitializationContext();
+ context.setSubClusterPolicyConfiguration(fpc);
+ context
+ .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ context.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster"));
+
+ // based on the "context" created instantiate new class and use it
+ Class c = Class.forName(wfp.getClass().getCanonicalName());
+ FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance();
+
+ FederationAMRMProxyPolicy federationAMRMProxyPolicy =
+ wfp2.getAMRMPolicy(context, null);
+
+ //needed only for tests (getARMRMPolicy change the "type" in conf)
+ fpc.setType(wfp.getClass().getCanonicalName());
+
+ FederationRouterPolicy federationRouterPolicy =
+ wfp2.getRouterPolicy(context, null);
+
+ Assert.assertEquals(federationAMRMProxyPolicy.getClass(),
+ expAMRMProxyPolicy);
+
+ Assert.assertEquals(federationRouterPolicy.getClass(),
+ expRouterPolicy);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index c79fd2a..d906b92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMR
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
@@ -38,6 +39,7 @@ public class TestFederationPolicyInitializationContextValidator {
private SubClusterPolicyConfiguration goodConfig;
private SubClusterResolver goodSR;
private FederationStateStoreFacade goodFacade;
+ private SubClusterId goodHome;
private FederationPolicyInitializationContext context;
@Before
@@ -45,8 +47,9 @@ public class TestFederationPolicyInitializationContextValidator {
goodFacade = FederationPoliciesTestUtil.initFacade();
goodConfig = new MockPolicyManager().serializeConf();
goodSR = FederationPoliciesTestUtil.initResolver();
+ goodHome = SubClusterId.newInstance("homesubcluster");
context = new FederationPolicyInitializationContext(goodConfig, goodSR,
- goodFacade);
+ goodFacade, goodHome);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..542a5ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link UniformBroadcastPolicyManager}.
+ */
+public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ //config policy
+ wfp = new UniformBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = UniformBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = UniformRandomRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..ab9cec4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple test of {@link WeightedLocalityPolicyManager}.
+ */
+public class TestWeightedLocalityPolicyManager extends
+ BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new WeightedLocalityPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+ amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setAMRMPolicyWeights(amrmWeights);
+
+ ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo(
+ policyInfo);
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = WeightedLocalityPolicyManager.class;
+ expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ expectedRouterPolicy = WeightedRandomRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy,
+ expectedRouterPolicy);
+
+ //check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(((WeightedLocalityPolicyManager) wfp)
+ .getWeightedPolicyInfo(), policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/112b99b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index 87ed8d1..85fdc96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -143,7 +143,7 @@ public final class FederationPoliciesTestUtil {
SubClusterInfo> activeSubclusters) throws YarnException {
FederationPolicyInitializationContext context =
new FederationPolicyInitializationContext(null, initResolver(),
- initFacade());
+ initFacade(), SubClusterId.newInstance("homesubcluster"));
initializePolicyContext(context, policy, policyInfo, activeSubclusters);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-5406. In-memory based
implementation of the FederationMembershipStateStore. Contributed by Ellen
Hui.
Posted by su...@apache.org.
YARN-5406. In-memory based implementation of the FederationMembershipStateStore. Contributed by Ellen Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbb8884f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbb8884f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbb8884f
Branch: refs/heads/YARN-2915
Commit: bbb8884fac8629d1ddbd3e99820692aca0581fd0
Parents: 00b32df
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Aug 4 15:54:38 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../store/impl/MemoryFederationStateStore.java | 138 ++++++++++++
.../federation/store/impl/package-info.java | 17 ++
.../records/GetSubClustersInfoRequest.java | 4 +
.../store/records/SubClusterState.java | 4 +
.../impl/FederationStateStoreBaseTest.java | 221 +++++++++++++++++++
.../impl/TestMemoryFederationStateStore.java | 49 ++++
6 files changed, 433 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbb8884f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
new file mode 100644
index 0000000..7fdc4a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * In-memory implementation of FederationMembershipStateStore.
+ */
+public class MemoryFederationStateStore
+ implements FederationMembershipStateStore {
+
+ private final Map<SubClusterId, SubClusterInfo> membership =
+ new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
+ private final MonotonicClock clock = new MonotonicClock();
+
+ @Override
+ public Version getMembershipStateStoreVersion() {
+ return null;
+ }
+
+ @Override
+ public SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest request) throws YarnException {
+ SubClusterInfo subClusterInfo = request.getSubClusterInfo();
+ subClusterInfo.setLastStartTime(clock.getTime());
+ membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
+ return SubClusterRegisterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest request) throws YarnException {
+ SubClusterInfo subClusterInfo = membership.get(request.getSubClusterId());
+ if (subClusterInfo == null) {
+ throw new YarnException(
+ "SubCluster " + request.getSubClusterId().toString() + " not found");
+ } else {
+ subClusterInfo.setState(request.getState());
+ }
+
+ return SubClusterDeregisterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest request) throws YarnException {
+
+ SubClusterId subClusterId = request.getSubClusterId();
+ SubClusterInfo subClusterInfo = membership.get(subClusterId);
+
+ if (subClusterInfo == null) {
+ throw new YarnException("Subcluster " + subClusterId.toString()
+ + " does not exist; cannot heartbeat");
+ }
+
+ subClusterInfo.setLastHeartBeat(clock.getTime());
+ subClusterInfo.setState(request.getState());
+ subClusterInfo.setCapability(request.getCapability());
+
+ return SubClusterHeartbeatResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest request) throws YarnException {
+ SubClusterId subClusterId = request.getSubClusterId();
+ if (!membership.containsKey(subClusterId)) {
+ throw new YarnException(
+ "Subcluster " + subClusterId.toString() + " does not exist");
+ }
+
+ return GetSubClusterInfoResponse.newInstance(membership.get(subClusterId));
+ }
+
+ @Override
+ public GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest request) throws YarnException {
+ List<SubClusterInfo> result = new ArrayList<SubClusterInfo>();
+
+ for (SubClusterInfo info : membership.values()) {
+ if (!request.getFilterInactiveSubClusters()
+ || info.getState().isActive()) {
+ result.add(info);
+ }
+ }
+ return GetSubClustersInfoResponse.newInstance(result);
+
+ }
+
+ @VisibleForTesting
+ public Map<SubClusterId, SubClusterInfo> getMembershipTable() {
+ return membership;
+ }
+
+ @VisibleForTesting
+ public void clearMembershipTable() {
+ membership.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbb8884f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java
new file mode 100644
index 0000000..56e1274
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.impl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbb8884f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
index 3264d81..90d2f99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
@@ -26,6 +26,10 @@ import org.apache.hadoop.yarn.util.Records;
/**
* Request class to obtain information about all sub-clusters that are
* participating in federation.
+ *
+ * If filterInactiveSubClusters is set to true, only active sub-clusters will be
+ * returned; otherwise, all sub-clusters will be returned regardless of state.
+ * By default, filterInactiveSubClusters is true.
*/
@Private
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbb8884f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
index 22cec99..ff49aaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
@@ -53,6 +53,10 @@ public enum SubClusterState {
return (this != SC_RUNNING && this != SC_NEW);
}
+ public boolean isActive() {
+ return this == SC_RUNNING;
+ }
+
public boolean isFinal() {
return (this == SC_UNREGISTERED || this == SC_DECOMMISSIONED
|| this == SC_LOST);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbb8884f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
new file mode 100644
index 0000000..7eb1c86
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.io.IOException;
+
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Base class for FederationMembershipStateStore implementations.
+ */
+public abstract class FederationStateStoreBaseTest {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreBaseTest.class);
+ private static final MonotonicClock CLOCK = new MonotonicClock();
+
+ private FederationMembershipStateStore stateStore = getStateStore();
+
+ @Before
+ public void before() throws IOException {
+ clearMembership();
+ }
+
+ @Test
+ public void testRegisterSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ SubClusterRegisterResponse result = stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ Map<SubClusterId, SubClusterInfo> membership = getMembership();
+
+ Assert.assertNotNull(membership.get(subClusterId));
+ Assert.assertNotNull(result);
+ Assert.assertEquals(subClusterInfo, membership.get(subClusterId));
+ }
+
+ @Test
+ public void testDeregisterSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+
+ SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
+
+ stateStore.deregisterSubCluster(deregisterRequest);
+
+ Map<SubClusterId, SubClusterInfo> membership = getMembership();
+ Assert.assertNotNull(membership.get(subClusterId));
+ Assert.assertEquals(membership.get(subClusterId).getState(),
+ SubClusterState.SC_UNREGISTERED);
+ }
+
+ @Test
+ public void testDeregisterSubClusterUnknownSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+
+ SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
+ try {
+ stateStore.deregisterSubCluster(deregisterRequest);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().startsWith("SubCluster SC not found"));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterInfo() throws Exception {
+
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ Assert.assertEquals(subClusterInfo,
+ stateStore.getSubCluster(request).getSubClusterInfo());
+ }
+
+ @Test
+ public void testGetSubClusterInfoUnknownSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+
+ try {
+ stateStore.getSubCluster(request).getSubClusterInfo();
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Subcluster SC does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetAllSubClustersInfo() throws Exception {
+
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ SubClusterInfo subClusterInfo1 = createSubClusterInfo(subClusterId1);
+
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ SubClusterInfo subClusterInfo2 = createSubClusterInfo(subClusterId2);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo1));
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo2));
+
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
+ .newInstance(subClusterId1, SubClusterState.SC_RUNNING, ""));
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
+ .newInstance(subClusterId2, SubClusterState.SC_UNHEALTHY, ""));
+
+ Assert.assertTrue(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(true))
+ .getSubClusters().contains(subClusterInfo1));
+ Assert.assertFalse(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(true))
+ .getSubClusters().contains(subClusterInfo2));
+
+ Assert.assertTrue(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(false))
+ .getSubClusters().contains(subClusterInfo1));
+ Assert.assertTrue(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(false))
+ .getSubClusters().contains(subClusterInfo2));
+ }
+
+ @Test
+ public void testSubClusterHeartbeat() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+
+ SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+ stateStore.subClusterHeartbeat(heartbeatRequest);
+
+ Map<SubClusterId, SubClusterInfo> membership = getMembership();
+ Assert.assertEquals(membership.get(subClusterId).getState(),
+ SubClusterState.SC_RUNNING);
+ Assert.assertNotNull(membership.get(subClusterId).getLastHeartBeat());
+ }
+
+ @Test
+ public void testSubClusterHeartbeatUnknownSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+
+ try {
+ stateStore.subClusterHeartbeat(heartbeatRequest);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Subcluster SC does not exist; cannot heartbeat"));
+ }
+ }
+
+ private SubClusterInfo createSubClusterInfo(SubClusterId subClusterId) {
+
+ String amRMAddress = "1.2.3.4:1";
+ String clientRMAddress = "1.2.3.4:2";
+ String rmAdminAddress = "1.2.3.4:3";
+ String webAppAddress = "1.2.3.4:4";
+
+ return SubClusterInfo.newInstance(subClusterId, amRMAddress,
+ clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
+ CLOCK.getTime(), "cabability");
+ }
+
+ protected abstract Map<SubClusterId, SubClusterInfo> getMembership();
+
+ protected abstract void clearMembership();
+
+ protected abstract FederationMembershipStateStore getStateStore();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbb8884f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
new file mode 100644
index 0000000..b74ffbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * Unit tests for MemoryFederationStateStore.
+ */
+public class TestMemoryFederationStateStore
+ extends FederationStateStoreBaseTest {
+
+ private static final MemoryFederationStateStore STATESTORE =
+ new MemoryFederationStateStore();
+
+ @Override
+ protected Map<SubClusterId, SubClusterInfo> getMembership() {
+ return STATESTORE.getMembershipTable();
+ }
+
+ @Override
+ protected void clearMembership() {
+ STATESTORE.clearMembershipTable();
+ }
+
+ @Override
+ protected FederationMembershipStateStore getStateStore() {
+ return STATESTORE;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDFS-11403. Zookeper ACLs on NN HA
enabled clusters should be handled consistently. Contributed by Hanisha
Koneru.
Posted by su...@apache.org.
HDFS-11403. Zookeper ACLs on NN HA enabled clusters should be handled consistently. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0aacd8fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0aacd8fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0aacd8fd
Branch: refs/heads/YARN-2915
Commit: 0aacd8fd2530f9f5febbe81ec05cd958cc0c3e2c
Parents: 07a5184
Author: Arpit Agarwal <ar...@apache.org>
Authored: Sat Feb 11 01:17:56 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Sat Feb 11 01:17:56 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/ha/ActiveStandbyElector.java | 37 ++++++++++++++++++--
.../hadoop/ha/TestActiveStandbyElector.java | 31 ++++++++++++++++
2 files changed, 65 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aacd8fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 9a90ccf..db853f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -346,8 +346,13 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
createWithRetries(prefixPath, new byte[]{}, zkAcl, CreateMode.PERSISTENT);
} catch (KeeperException e) {
if (isNodeExists(e.code())) {
- // This is OK - just ensuring existence.
- continue;
+ // Set ACLs for parent node, if they do not exist or are different
+ try {
+ setAclsWithRetries(prefixPath);
+ } catch (KeeperException e1) {
+ throw new IOException("Couldn't set ACLs on parent ZNode: " +
+ prefixPath, e1);
+ }
} else {
throw new IOException("Couldn't create " + prefixPath, e);
}
@@ -1066,14 +1071,36 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
});
}
+ private void setAclsWithRetries(final String path)
+ throws KeeperException, InterruptedException {
+ Stat stat = new Stat();
+ zkDoWithRetries(new ZKAction<Void>() {
+ @Override
+ public Void run() throws KeeperException, InterruptedException {
+ List<ACL> acl = zkClient.getACL(path, stat);
+ if (acl == null || !acl.containsAll(zkAcl) ||
+ !zkAcl.containsAll(acl)) {
+ zkClient.setACL(path, zkAcl, stat.getVersion());
+ }
+ return null;
+ }
+ }, Code.BADVERSION);
+ }
+
private <T> T zkDoWithRetries(ZKAction<T> action) throws KeeperException,
InterruptedException {
+ return zkDoWithRetries(action, null);
+ }
+
+ private <T> T zkDoWithRetries(ZKAction<T> action, Code retryCode)
+ throws KeeperException, InterruptedException {
int retry = 0;
while (true) {
try {
return action.run();
} catch (KeeperException ke) {
- if (shouldRetry(ke.code()) && ++retry < maxRetryNum) {
+ if ((shouldRetry(ke.code()) || shouldRetry(ke.code(), retryCode))
+ && ++retry < maxRetryNum) {
continue;
}
throw ke;
@@ -1189,6 +1216,10 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
private static boolean shouldRetry(Code code) {
return code == Code.CONNECTIONLOSS || code == Code.OPERATIONTIMEOUT;
}
+
+ private static boolean shouldRetry(Code code, Code retryIfCode) {
+ return (retryIfCode == null ? false : retryIfCode == code);
+ }
@Override
public String toString() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0aacd8fd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
index 8c6f7a3..ca3389f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElector.java
@@ -715,6 +715,37 @@ public class TestActiveStandbyElector {
}
}
}
+
+ /**
+ * Test that ACLs are set on parent zNode even if the node already exists.
+ */
+ @Test
+ public void testParentZNodeACLs() throws Exception {
+ KeeperException ke = new KeeperException(Code.NODEEXISTS) {
+ @Override
+ public Code code() {
+ return super.code();
+ }
+ };
+
+ Mockito.when(mockZK.create(Mockito.anyString(), Mockito.eq(new byte[]{}),
+ Mockito.anyListOf(ACL.class),
+ Mockito.eq(CreateMode.PERSISTENT))).thenThrow(ke);
+
+ elector.ensureParentZNode();
+
+ StringBuilder prefix = new StringBuilder();
+ for (String part : ZK_PARENT_NAME.split("/")) {
+ if (part.isEmpty()) continue;
+ prefix.append("/").append(part);
+ if (!"/".equals(prefix.toString())) {
+ Mockito.verify(mockZK).getACL(Mockito.eq(prefix.toString()),
+ Mockito.eq(new Stat()));
+ Mockito.verify(mockZK).setACL(Mockito.eq(prefix.toString()),
+ Mockito.eq(Ids.OPEN_ACL_UNSAFE), Mockito.anyInt());
+ }
+ }
+ }
/**
* Test for a bug encountered during development of HADOOP-8163:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-5307. Federation Application
State Store internal APIs
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..8b72a1e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationsHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetApplicationsHomeSubClusterResponsePBImpl
+ extends GetApplicationsHomeSubClusterResponse {
+
+ private GetApplicationsHomeSubClusterResponseProto proto =
+ GetApplicationsHomeSubClusterResponseProto.getDefaultInstance();
+ private GetApplicationsHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List<ApplicationHomeSubCluster> appsHomeSubCluster;
+
+ public GetApplicationsHomeSubClusterResponsePBImpl() {
+ builder = GetApplicationsHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public GetApplicationsHomeSubClusterResponsePBImpl(
+ GetApplicationsHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationsHomeSubClusterResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetApplicationsHomeSubClusterResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.appsHomeSubCluster != null) {
+ addSubClustersInfoToProto();
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public List<ApplicationHomeSubCluster> getAppsHomeSubClusters() {
+ initSubClustersInfoList();
+ return appsHomeSubCluster;
+ }
+
+ @Override
+ public void setAppsHomeSubClusters(
+ List<ApplicationHomeSubCluster> appsHomeSubClusters) {
+ maybeInitBuilder();
+ if (appsHomeSubClusters == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ this.appsHomeSubCluster = appsHomeSubClusters;
+ }
+
+ private void initSubClustersInfoList() {
+ if (this.appsHomeSubCluster != null) {
+ return;
+ }
+ GetApplicationsHomeSubClusterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<ApplicationHomeSubClusterProto> subClusterInfosList =
+ p.getAppSubclusterMapList();
+ appsHomeSubCluster = new ArrayList<ApplicationHomeSubCluster>();
+
+ for (ApplicationHomeSubClusterProto r : subClusterInfosList) {
+ appsHomeSubCluster.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addSubClustersInfoToProto() {
+ maybeInitBuilder();
+ builder.clearAppSubclusterMap();
+ if (appsHomeSubCluster == null) {
+ return;
+ }
+ Iterable<ApplicationHomeSubClusterProto> iterable =
+ new Iterable<ApplicationHomeSubClusterProto>() {
+ @Override
+ public Iterator<ApplicationHomeSubClusterProto> iterator() {
+ return new Iterator<ApplicationHomeSubClusterProto>() {
+
+ private Iterator<ApplicationHomeSubCluster> iter =
+ appsHomeSubCluster.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ApplicationHomeSubClusterProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllAppSubclusterMap(iterable);
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
index d39ef7f..92bdf06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..e42eb00
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link UpdateApplicationHomeSubClusterRequest} .
+ */
+@Private
+@Unstable
+public class UpdateApplicationHomeSubClusterRequestPBImpl
+ extends UpdateApplicationHomeSubClusterRequest {
+
+ private UpdateApplicationHomeSubClusterRequestProto proto =
+ UpdateApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private UpdateApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public UpdateApplicationHomeSubClusterRequestPBImpl() {
+ builder = UpdateApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public UpdateApplicationHomeSubClusterRequestPBImpl(
+ UpdateApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public UpdateApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = UpdateApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationHomeSubCluster getApplicationHomeSubCluster() {
+ UpdateApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasAppSubclusterMap()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAppSubclusterMap());
+ }
+
+ @Override
+ public void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationInfo) {
+ maybeInitBuilder();
+ if (applicationInfo == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ builder.setAppSubclusterMap(convertToProtoFormat(applicationInfo));
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..ec31f0b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link UpdateApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class UpdateApplicationHomeSubClusterResponsePBImpl
+ extends UpdateApplicationHomeSubClusterResponse {
+
+ private UpdateApplicationHomeSubClusterResponseProto proto =
+ UpdateApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private UpdateApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public UpdateApplicationHomeSubClusterResponsePBImpl() {
+ builder = UpdateApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public UpdateApplicationHomeSubClusterResponsePBImpl(
+ UpdateApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public UpdateApplicationHomeSubClusterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 1b2e53e..b1ad310 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -90,4 +90,47 @@ message GetSubClustersInfoRequestProto {
message GetSubClustersInfoResponseProto {
repeated SubClusterInfoProto sub_cluster_infos = 1;
-}
\ No newline at end of file
+}
+
+message ApplicationHomeSubClusterProto {
+ optional ApplicationIdProto application_id = 1;
+ optional SubClusterIdProto home_sub_cluster = 2;
+}
+
+message AddApplicationHomeSubClusterRequestProto {
+ optional ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message AddApplicationHomeSubClusterResponseProto {
+}
+
+message UpdateApplicationHomeSubClusterRequestProto {
+ optional ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message UpdateApplicationHomeSubClusterResponseProto {
+}
+
+message GetApplicationHomeSubClusterRequestProto {
+ optional ApplicationIdProto application_id = 1;
+}
+
+message GetApplicationHomeSubClusterResponseProto {
+ optional ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message GetApplicationsHomeSubClusterRequestProto {
+
+}
+
+message GetApplicationsHomeSubClusterResponseProto {
+ repeated ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+
+message DeleteApplicationHomeSubClusterRequestProto {
+ optional ApplicationIdProto application_id = 1;
+}
+
+message DeleteApplicationHomeSubClusterResponseProto {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c503284/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
index 681edb1..210a246 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -19,6 +19,14 @@ package org.apache.hadoop.yarn.server.federation.store.records;
import org.apache.hadoop.yarn.api.BasePBImplRecordsTest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
@@ -31,6 +39,16 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClu
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationsHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationsHomeSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoRequestPBImpl;
@@ -43,6 +61,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubCluster
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.records.Version;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -58,6 +78,7 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
generateByNewInstance(Version.class);
generateByNewInstance(SubClusterId.class);
generateByNewInstance(SubClusterInfo.class);
+ generateByNewInstance(ApplicationHomeSubCluster.class);
}
@Test
@@ -130,4 +151,64 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
GetSubClustersInfoResponseProto.class);
}
+ @Test
+ public void testAddApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(AddApplicationHomeSubClusterRequestPBImpl.class,
+ AddApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testAddApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(AddApplicationHomeSubClusterResponsePBImpl.class,
+ AddApplicationHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(UpdateApplicationHomeSubClusterRequestPBImpl.class,
+ UpdateApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(UpdateApplicationHomeSubClusterResponsePBImpl.class,
+ UpdateApplicationHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(GetApplicationHomeSubClusterRequestPBImpl.class,
+ GetApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(GetApplicationHomeSubClusterResponsePBImpl.class,
+ GetApplicationHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testGetApplicationsHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(GetApplicationsHomeSubClusterRequestPBImpl.class,
+ GetApplicationsHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testGetApplicationsHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(GetApplicationsHomeSubClusterResponsePBImpl.class,
+ GetApplicationsHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(DeleteApplicationHomeSubClusterRequestPBImpl.class,
+ DeleteApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(DeleteApplicationHomeSubClusterResponsePBImpl.class,
+ DeleteApplicationHomeSubClusterResponseProto.class);
+ }
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-13769. AliyunOSS: update
oss sdk version. Contributed by Genmao Yu
Posted by su...@apache.org.
HADOOP-13769. AliyunOSS: update oss sdk version. Contributed by Genmao Yu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/243c0f33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/243c0f33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/243c0f33
Branch: refs/heads/YARN-2915
Commit: 243c0f33ec2559c1b727f6c7ea73625df3ac3a43
Parents: 01be450
Author: Kai Zheng <ka...@intel.com>
Authored: Mon Feb 13 13:21:44 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Mon Feb 13 13:21:44 2017 +0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/243c0f33/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d3579a1..606f7fc 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1104,7 +1104,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
- <version>2.2.1</version>
+ <version>2.4.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-5324. Stateless Federation
router policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cf5730c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index 8c2115b..f901329 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -19,13 +19,20 @@ package org.apache.hadoop.yarn.server.federation.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
-import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.*;
import java.net.URL;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
@@ -41,6 +48,41 @@ public final class FederationPoliciesTestUtil {
// disabled.
}
+
+ public static void initializePolicyContext(
+ FederationPolicyInitializationContext fpc, ConfigurableFederationPolicy
+ policy, WeightedPolicyInfo policyInfo,
+ Map<SubClusterId, SubClusterInfo> activeSubclusters)
+ throws YarnException {
+ ByteBuffer buf = policyInfo.toByteBuffer();
+ fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+ .newInstance("queue1", policy.getClass().getCanonicalName(), buf));
+ FederationStateStoreFacade facade = FederationStateStoreFacade
+ .getInstance();
+ FederationStateStore fss = mock(FederationStateStore.class);
+
+ if (activeSubclusters == null) {
+ activeSubclusters = new HashMap<SubClusterId, SubClusterInfo>();
+ }
+ GetSubClustersInfoResponse response = GetSubClustersInfoResponse
+ .newInstance(new ArrayList<SubClusterInfo>(activeSubclusters.values()));
+
+ when(fss.getSubClusters(any())).thenReturn(response);
+ facade.reinitialize(fss, new Configuration());
+ fpc.setFederationStateStoreFacade(facade);
+ policy.reinitialize(fpc);
+ }
+
+ public static void initializePolicyContext(
+ ConfigurableFederationPolicy policy,
+ WeightedPolicyInfo policyInfo, Map<SubClusterId,
+ SubClusterInfo> activeSubclusters) throws YarnException {
+ FederationPolicyInitializationContext context =
+ new FederationPolicyInitializationContext(null, initResolver(),
+ initFacade());
+ initializePolicyContext(context, policy, policyInfo, activeSubclusters);
+ }
+
/**
* Initialize a {@link SubClusterResolver}.
*
@@ -66,18 +108,52 @@ public final class FederationPoliciesTestUtil {
* Initialiaze a main-memory {@link FederationStateStoreFacade} used for
* testing, wiht a mock resolver.
*
+ * @param subClusterInfos the list of subclusters to be served on
+ * getSubClusters invocations.
+ *
* @return the facade.
*
* @throws YarnException in case the initialization is not successful.
*/
- public static FederationStateStoreFacade initFacade() throws YarnException {
+
+ public static FederationStateStoreFacade initFacade(
+ List<SubClusterInfo> subClusterInfos, SubClusterPolicyConfiguration
+ policyConfiguration) throws YarnException {
FederationStateStoreFacade goodFacade = FederationStateStoreFacade
.getInstance();
FederationStateStore fss = mock(FederationStateStore.class);
GetSubClustersInfoResponse response = GetSubClustersInfoResponse
- .newInstance(new ArrayList<>());
+ .newInstance(subClusterInfos);
when(fss.getSubClusters(any())).thenReturn(response);
+
+ List<SubClusterPolicyConfiguration> configurations = new ArrayList<>();
+ configurations.add(policyConfiguration);
+
+ GetSubClusterPoliciesConfigurationsResponse policiesResponse =
+ GetSubClusterPoliciesConfigurationsResponse
+ .newInstance(configurations);
+ when(fss.getPoliciesConfigurations(any())).thenReturn(policiesResponse);
+
+ GetSubClusterPolicyConfigurationResponse policyResponse =
+ GetSubClusterPolicyConfigurationResponse
+ .newInstance(policyConfiguration);
+ when(fss.getPolicyConfiguration(any())).thenReturn(policyResponse);
+
goodFacade.reinitialize(fss, new Configuration());
return goodFacade;
}
+
+ /**
+ * Initialiaze a main-memory {@link FederationStateStoreFacade} used for
+ * testing, wiht a mock resolver.
+ *
+ * @return the facade.
+ *
+ * @throws YarnException in case the initialization is not successful.
+ */
+ public static FederationStateStoreFacade initFacade() throws YarnException {
+ return initFacade(new ArrayList<>(), mock(SubClusterPolicyConfiguration
+ .class));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HADOOP-13233. help of stat is
confusing. Contributed by Attila Bukor.
Posted by su...@apache.org.
HADOOP-13233. help of stat is confusing. Contributed by Attila Bukor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc45da79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc45da79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc45da79
Branch: refs/heads/YARN-2915
Commit: cc45da79fda7dfba2795ac397d62f40a858dcdd9
Parents: 464ff47
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Feb 13 10:14:45 2017 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Feb 13 10:14:45 2017 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/shell/Stat.java | 4 ++--
.../hadoop-common/src/site/markdown/FileSystemShell.md | 2 +-
.../hadoop-common/src/test/resources/testConf.xml | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc45da79/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
index 42f7843..cf8270e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.FileStatus;
* Format sequences:<br>
* %a: Permissions in octal<br>
* %A: Permissions in symbolic style<br>
- * %b: Size of file in blocks<br>
+ * %b: Size of file in bytes<br>
* %F: Type<br>
* %g: Group name of owner<br>
* %n: Filename<br>
@@ -60,7 +60,7 @@ class Stat extends FsCommand {
"Print statistics about the file/directory at <path>" + NEWLINE +
"in the specified format. Format accepts permissions in" + NEWLINE +
"octal (%a) and symbolic (%A), filesize in" + NEWLINE +
- "blocks (%b), type (%F), group name of owner (%g)," + NEWLINE +
+ "bytes (%b), type (%F), group name of owner (%g)," + NEWLINE +
"name (%n), block size (%o), replication (%r), user name" + NEWLINE +
"of owner (%u), modification date (%y, %Y)." + NEWLINE +
"%y shows UTC date as \"yyyy-MM-dd HH:mm:ss\" and" + NEWLINE +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc45da79/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 43fc28b..42fddc9 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -667,7 +667,7 @@ stat
Usage: `hadoop fs -stat [format] <path> ...`
-Print statistics about the file/directory at \<path\> in the specified format. Format accepts permissions in octal (%a) and symbolic (%A), filesize in blocks (%b), type (%F), group name of owner (%g), name (%n), block size (%o), replication (%r), user name of owner(%u), and modification date (%y, %Y). %y shows UTC date as "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC. If the format is not specified, %y is used by default.
+Print statistics about the file/directory at \<path\> in the specified format. Format accepts permissions in octal (%a) and symbolic (%A), filesize in bytes (%b), type (%F), group name of owner (%g), name (%n), block size (%o), replication (%r), user name of owner(%u), and modification date (%y, %Y). %y shows UTC date as "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC. If the format is not specified, %y is used by default.
Example:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc45da79/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index d285f33..112aea0 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -867,7 +867,7 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*blocks \(%b\), type \(%F\), group name of owner \(%g\),( )*</expected-output>
+ <expected-output>^( |\t)*bytes \(%b\), type \(%F\), group name of owner \(%g\),( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-3672. Create Facade for
Federation State and Policy Store. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-3672. Create Facade for Federation State and Policy Store. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ea752f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ea752f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ea752f2
Branch: refs/heads/YARN-2915
Commit: 9ea752f237c30288d7ef397f29ff43244698b511
Parents: 0b0386e
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 17 11:13:19 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 13 +
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../src/main/resources/yarn-default.xml | 20 +-
.../hadoop-yarn-server-common/pom.xml | 10 +
.../utils/FederationStateStoreFacade.java | 532 +++++++++++++++++++
.../server/federation/utils/package-info.java | 17 +
.../utils/FederationStateStoreTestUtil.java | 149 ++++++
.../utils/TestFederationStateStoreFacade.java | 148 ++++++
9 files changed, 905 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 606f7fc..080c812 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -93,6 +93,9 @@
<apacheds.version>2.0.0-M21</apacheds.version>
<ldap-api.version>1.0.0-M33</ldap-api.version>
+ <jcache.version>1.0.0</jcache.version>
+ <ehcache.version>3.0.3</ehcache.version>
+
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -1245,6 +1248,16 @@
<artifactId>kerb-simplekdc</artifactId>
<version>1.0.0-RC2</version>
</dependency>
+ <dependency>
+ <groupId>javax.cache</groupId>
+ <artifactId>cache-api</artifactId>
+ <version>${jcache.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>${ehcache.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a047450..72b0d26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2486,6 +2486,19 @@ public class YarnConfiguration extends Configuration {
////////////////////////////////
public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+
+ public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
+ FEDERATION_PREFIX + "state-store.class";
+
+ public static final String DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS =
+ "org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore";
+
+ public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
+ FEDERATION_PREFIX + "cache-ttl.secs";
+
+ // 5 minutes
+ public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3da4bab..bfc2534 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -68,6 +68,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
+ // Federation default configs to be ignored
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
+
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
configurationPropsToSkipCompare.add(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 31c86f8..bc244c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2608,8 +2608,8 @@
<description>The arguments to pass to the Node label script.</description>
<name>yarn.nodemanager.node-labels.provider.script.opts</name>
</property>
- <!-- Other Configuration -->
+ <!-- Federation Configuration -->
<property>
<description>
Machine list file to be loaded by the FederationSubCluster Resolver
@@ -2618,6 +2618,24 @@
</property>
<property>
+ <description>
+ Store class name for federation state store
+ </description>
+ <name>yarn.federation.state-store.class</name>
+ <value>org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore</value>
+ </property>
+
+ <property>
+ <description>
+ The time in seconds after which the federation state store local cache
+ will be refreshed periodically
+ </description>
+ <name>yarn.federation.cache-ttl.secs</name>
+ <value>300</value>
+ </property>
+
+ <!-- Other Configuration -->
+ <property>
<description>The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 6d2fbef..cfb2533 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -108,6 +108,16 @@
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>javax.cache</groupId>
+ <artifactId>cache-api</artifactId>
+ <version>${jcache.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>${ehcache.version}</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
new file mode 100644
index 0000000..f1c8218
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -0,0 +1,532 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import javax.cache.Cache;
+import javax.cache.CacheManager;
+import javax.cache.Caching;
+import javax.cache.configuration.CompleteConfiguration;
+import javax.cache.configuration.FactoryBuilder;
+import javax.cache.configuration.MutableConfiguration;
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.integration.CacheLoader;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.spi.CachingProvider;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ *
+ * The FederationStateStoreFacade is an utility wrapper that provides singleton
+ * access to the Federation state store. It abstracts out retries and in
+ * addition, it also implements the caching for various objects.
+ *
+ */
+public final class FederationStateStoreFacade {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreFacade.class);
+
+ private static final String GET_SUBCLUSTERS_CACHEID = "getSubClusters";
+ private static final String GET_POLICIES_CONFIGURATIONS_CACHEID =
+ "getPoliciesConfigurations";
+
+ private static final FederationStateStoreFacade FACADE =
+ new FederationStateStoreFacade();
+
+ private FederationStateStore stateStore;
+ private int cacheTimeToLive;
+ private Configuration conf;
+ private Cache<Object, Object> cache;
+
+ private FederationStateStoreFacade() {
+ initializeFacadeInternal(new Configuration());
+ }
+
+ private void initializeFacadeInternal(Configuration config) {
+ this.conf = config;
+ try {
+ this.stateStore = (FederationStateStore) createRetryInstance(this.conf,
+ YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS,
+ FederationStateStore.class, createRetryPolicy(conf));
+ this.stateStore.init(conf);
+
+ initCache();
+
+ } catch (YarnException ex) {
+ LOG.error("Failed to initialize the FederationStateStoreFacade object",
+ ex);
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * Delete and re-initialize the cache, to force it to use the given
+ * configuration.
+ *
+ * @param store the {@link FederationStateStore} instance to reinitialize with
+ * @param config the updated configuration to reinitialize with
+ */
+ @VisibleForTesting
+ public synchronized void reinitialize(FederationStateStore store,
+ Configuration config) {
+ this.conf = config;
+ this.stateStore = store;
+ clearCache();
+ initCache();
+ }
+
+ public static RetryPolicy createRetryPolicy(Configuration conf) {
+ // Retry settings for StateStore
+ RetryPolicy retryPolicy = RetryPolicies.exponentialBackoffRetry(
+ conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, Integer.SIZE),
+ conf.getLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
+ YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS),
+ TimeUnit.MILLISECONDS);
+
+ return retryPolicy;
+ }
+
+ private boolean isCachingEnabled() {
+ return (cacheTimeToLive > 0);
+ }
+
+ private void initCache() {
+ // Picking the JCache provider from classpath, need to make sure there's
+ // no conflict or pick up a specific one in the future
+ cacheTimeToLive =
+ conf.getInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS,
+ YarnConfiguration.DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS);
+ if (isCachingEnabled()) {
+ CachingProvider jcacheProvider = Caching.getCachingProvider();
+ CacheManager jcacheManager = jcacheProvider.getCacheManager();
+ this.cache = jcacheManager.getCache(this.getClass().getSimpleName());
+ if (this.cache == null) {
+ LOG.info("Creating a JCache Manager with name "
+ + this.getClass().getSimpleName());
+ Duration cacheExpiry = new Duration(TimeUnit.SECONDS, cacheTimeToLive);
+ CompleteConfiguration<Object, Object> configuration =
+ new MutableConfiguration<Object, Object>().setStoreByValue(false)
+ .setReadThrough(true)
+ .setExpiryPolicyFactory(
+ new FactoryBuilder.SingletonFactory<ExpiryPolicy>(
+ new CreatedExpiryPolicy(cacheExpiry)))
+ .setCacheLoaderFactory(
+ new FactoryBuilder.SingletonFactory<CacheLoader<Object, Object>>(
+ new CacheLoaderImpl<Object, Object>()));
+ this.cache = jcacheManager.createCache(this.getClass().getSimpleName(),
+ configuration);
+ }
+ }
+ }
+
+ private void clearCache() {
+ CachingProvider jcacheProvider = Caching.getCachingProvider();
+ CacheManager jcacheManager = jcacheProvider.getCacheManager();
+
+ jcacheManager.destroyCache(this.getClass().getSimpleName());
+ this.cache = null;
+ }
+
+ /**
+ * Returns the singleton instance of the FederationStateStoreFacade object.
+ *
+ * @return the singleton {@link FederationStateStoreFacade} instance
+ */
+ public static FederationStateStoreFacade getInstance() {
+ return FACADE;
+ }
+
+ /**
+ * Returns the {@link SubClusterInfo} for the specified {@link SubClusterId}.
+ *
+ * @param subClusterId the identifier of the sub-cluster
+ * @return the sub cluster information
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterInfo getSubCluster(final SubClusterId subClusterId)
+ throws YarnException {
+ if (isCachingEnabled()) {
+ return getSubClusters(false).get(subClusterId);
+ } else {
+ return stateStore
+ .getSubCluster(GetSubClusterInfoRequest.newInstance(subClusterId))
+ .getSubClusterInfo();
+ }
+ }
+
+ /**
+ * Updates the cache with the central {@link FederationStateStore} and returns
+ * the {@link SubClusterInfo} for the specified {@link SubClusterId}.
+ *
+ * @param subClusterId the identifier of the sub-cluster
+ * @param flushCache flag to indicate if the cache should be flushed or not
+ * @return the sub cluster information
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterInfo getSubCluster(final SubClusterId subClusterId,
+ final boolean flushCache) throws YarnException {
+ if (flushCache && isCachingEnabled()) {
+ LOG.info("Flushing subClusters from cache and rehydrating from store,"
+ + " most likely on account of RM failover.");
+ cache.remove(buildGetSubClustersCacheRequest(false));
+ }
+ return getSubCluster(subClusterId);
+ }
+
+ /**
+ * Returns the {@link SubClusterInfo} of all active sub cluster(s).
+ *
+ * @param filterInactiveSubClusters whether to filter out inactive
+ * sub-clusters
+ * @return the information of all active sub cluster(s)
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ @SuppressWarnings("unchecked")
+ public Map<SubClusterId, SubClusterInfo> getSubClusters(
+ final boolean filterInactiveSubClusters) throws YarnException {
+ try {
+ if (isCachingEnabled()) {
+ return (Map<SubClusterId, SubClusterInfo>) cache
+ .get(buildGetSubClustersCacheRequest(filterInactiveSubClusters));
+ } else {
+ return buildSubClusterInfoMap(stateStore.getSubClusters(
+ GetSubClustersInfoRequest.newInstance(filterInactiveSubClusters)));
+ }
+ } catch (Throwable ex) {
+ throw new YarnException(ex);
+ }
+ }
+
+ /**
+ * Returns the {@link SubClusterPolicyConfiguration} for the specified queue.
+ *
+ * @param queue the queue whose policy is required
+ * @return the corresponding configured policy
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterPolicyConfiguration getPolicyConfiguration(
+ final String queue) throws YarnException {
+ if (isCachingEnabled()) {
+ return getPoliciesConfigurations().get(queue);
+ } else {
+ return stateStore
+ .getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue))
+ .getPolicyConfiguration();
+ }
+
+ }
+
+ /**
+ * Get the policies that is represented as
+ * {@link SubClusterPolicyConfiguration} for all currently active queues in
+ * the system.
+ *
+ * @return the policies for all currently active queues in the system
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ @SuppressWarnings("unchecked")
+ public Map<String, SubClusterPolicyConfiguration> getPoliciesConfigurations()
+ throws YarnException {
+ try {
+ if (isCachingEnabled()) {
+ return (Map<String, SubClusterPolicyConfiguration>) cache
+ .get(buildGetPoliciesConfigurationsCacheRequest());
+ } else {
+ return buildPolicyConfigMap(stateStore.getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest.newInstance()));
+ }
+ } catch (Throwable ex) {
+ throw new YarnException(ex);
+ }
+ }
+
+ /**
+ * Adds the home {@link SubClusterId} for the specified {@link ApplicationId}.
+ *
+ * @param appHomeSubCluster the mapping of the application to it's home
+ * sub-cluster
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public void addApplicationHomeSubCluster(
+ ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
+ stateStore.addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+ return;
+ }
+
+ /**
+ * Updates the home {@link SubClusterId} for the specified
+ * {@link ApplicationId}.
+ *
+ * @param appHomeSubCluster the mapping of the application to it's home
+ * sub-cluster
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public void updateApplicationHomeSubCluster(
+ ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
+ stateStore.updateApplicationHomeSubCluster(
+ UpdateApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+ return;
+ }
+
+ /**
+ * Returns the home {@link SubClusterId} for the specified
+ * {@link ApplicationId}.
+ *
+ * @param appId the identifier of the application
+ * @return the home sub cluster identifier
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterId getApplicationHomeSubCluster(ApplicationId appId)
+ throws YarnException {
+ GetApplicationHomeSubClusterResponse response =
+ stateStore.getApplicationHomeSubCluster(
+ GetApplicationHomeSubClusterRequest.newInstance(appId));
+ return response.getApplicationHomeSubCluster().getHomeSubCluster();
+ }
+
+ /**
+ * Helper method to create instances of Object using the class name defined in
+ * the configuration object. The instances creates {@link RetryProxy} using
+ * the specific {@link RetryPolicy}.
+ *
+ * @param conf the yarn configuration
+ * @param configuredClassName the configuration provider key
+ * @param defaultValue the default implementation for fallback
+ * @param type the class for which a retry proxy is required
+ * @param retryPolicy the policy for retrying method call failures
+ * @return a retry proxy for the specified interface
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> Object createRetryInstance(Configuration conf,
+ String configuredClassName, String defaultValue, Class<T> type,
+ RetryPolicy retryPolicy) {
+
+ String className = conf.get(configuredClassName, defaultValue);
+ try {
+ Class<?> clusterResolverClass = conf.getClassByName(className);
+ if (type.isAssignableFrom(clusterResolverClass)) {
+ return RetryProxy.create(type,
+ (T) ReflectionUtils.newInstance(clusterResolverClass, conf),
+ retryPolicy);
+ } else {
+ throw new YarnRuntimeException(
+ "Class: " + className + " not instance of " + type.getSimpleName());
+ }
+ } catch (Exception e) {
+ throw new YarnRuntimeException("Could not instantiate : " + className, e);
+ }
+ }
+
+ private Map<SubClusterId, SubClusterInfo> buildSubClusterInfoMap(
+ final GetSubClustersInfoResponse response) {
+ List<SubClusterInfo> subClusters = response.getSubClusters();
+ Map<SubClusterId, SubClusterInfo> subClustersMap =
+ new HashMap<>(subClusters.size());
+ for (SubClusterInfo subCluster : subClusters) {
+ subClustersMap.put(subCluster.getSubClusterId(), subCluster);
+ }
+ return subClustersMap;
+ }
+
+ private Object buildGetSubClustersCacheRequest(
+ final boolean filterInactiveSubClusters) {
+ final String cacheKey = buildCacheKey(getClass().getSimpleName(),
+ GET_SUBCLUSTERS_CACHEID, null);
+ CacheRequest<String, Map<SubClusterId, SubClusterInfo>> cacheRequest =
+ new CacheRequest<String, Map<SubClusterId, SubClusterInfo>>(cacheKey,
+ new Func<String, Map<SubClusterId, SubClusterInfo>>() {
+ @Override
+ public Map<SubClusterId, SubClusterInfo> invoke(String key)
+ throws Exception {
+ GetSubClustersInfoResponse subClusters =
+ stateStore.getSubClusters(GetSubClustersInfoRequest
+ .newInstance(filterInactiveSubClusters));
+ return buildSubClusterInfoMap(subClusters);
+ }
+ });
+ return cacheRequest;
+ }
+
+ private Map<String, SubClusterPolicyConfiguration> buildPolicyConfigMap(
+ GetSubClusterPoliciesConfigurationsResponse response) {
+ List<SubClusterPolicyConfiguration> policyConfigs =
+ response.getPoliciesConfigs();
+ Map<String, SubClusterPolicyConfiguration> queuePolicyConfigs =
+ new HashMap<>();
+ for (SubClusterPolicyConfiguration policyConfig : policyConfigs) {
+ queuePolicyConfigs.put(policyConfig.getQueue(), policyConfig);
+ }
+ return queuePolicyConfigs;
+ }
+
+ private Object buildGetPoliciesConfigurationsCacheRequest() {
+ final String cacheKey = buildCacheKey(getClass().getSimpleName(),
+ GET_POLICIES_CONFIGURATIONS_CACHEID, null);
+ CacheRequest<String, Map<String, SubClusterPolicyConfiguration>> cacheRequest =
+ new CacheRequest<String, Map<String, SubClusterPolicyConfiguration>>(
+ cacheKey,
+ new Func<String, Map<String, SubClusterPolicyConfiguration>>() {
+ @Override
+ public Map<String, SubClusterPolicyConfiguration> invoke(
+ String key) throws Exception {
+ GetSubClusterPoliciesConfigurationsResponse policyConfigs =
+ stateStore.getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest
+ .newInstance());
+ return buildPolicyConfigMap(policyConfigs);
+ }
+ });
+ return cacheRequest;
+ }
+
+ protected String buildCacheKey(String typeName, String methodName,
+ String argName) {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append(typeName).append(".");
+ buffer.append(methodName);
+ if (argName != null) {
+ buffer.append("::");
+ buffer.append(argName);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Internal class that implements the CacheLoader interface that can be
+ * plugged into the CacheManager to load objects into the cache for specified
+ * keys.
+ */
+ private static class CacheLoaderImpl<K, V> implements CacheLoader<K, V> {
+ @SuppressWarnings("unchecked")
+ @Override
+ public V load(K key) throws CacheLoaderException {
+ try {
+ CacheRequest<K, V> query = (CacheRequest<K, V>) key;
+ assert query != null;
+ return query.getValue();
+ } catch (Throwable ex) {
+ throw new CacheLoaderException(ex);
+ }
+ }
+
+ @Override
+ public Map<K, V> loadAll(Iterable<? extends K> keys)
+ throws CacheLoaderException {
+ // The FACADE does not use the Cache's getAll API. Hence this is not
+ // required to be implemented
+ throw new NotImplementedException();
+ }
+ }
+
+ /**
+ * Internal class that encapsulates the cache key and a function that returns
+ * the value for the specified key.
+ */
+ private static class CacheRequest<K, V> {
+ private K key;
+ private Func<K, V> func;
+
+ public CacheRequest(K key, Func<K, V> func) {
+ this.key = key;
+ this.func = func;
+ }
+
+ public V getValue() throws Exception {
+ return func.invoke(key);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((key == null) ? 0 : key.hashCode());
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ CacheRequest<K, V> other = (CacheRequest<K, V>) obj;
+ if (key == null) {
+ if (other.key != null) {
+ return false;
+ }
+ } else if (!key.equals(other.key)) {
+ return false;
+ }
+
+ return true;
+ }
+ }
+
+ /**
+ * Encapsulates a method that has one parameter and returns a value of the
+ * type specified by the TResult parameter.
+ */
+ protected interface Func<T, TResult> {
+ TResult invoke(T input) throws Exception;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java
new file mode 100644
index 0000000..39a46ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.utils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
new file mode 100644
index 0000000..c179521
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+/**
+ * Utility class for FederationStateStore unit tests.
+ */
+public class FederationStateStoreTestUtil {
+
+ private static final MonotonicClock CLOCK = new MonotonicClock();
+
+ public static final String SC_PREFIX = "SC-";
+ public static final String Q_PREFIX = "queue-";
+ public static final String POLICY_PREFIX = "policy-";
+
+ private FederationStateStore stateStore;
+
+ public FederationStateStoreTestUtil(FederationStateStore stateStore) {
+ this.stateStore = stateStore;
+ }
+
+ private SubClusterInfo createSubClusterInfo(SubClusterId subClusterId) {
+
+ String amRMAddress = "1.2.3.4:1";
+ String clientRMAddress = "1.2.3.4:2";
+ String rmAdminAddress = "1.2.3.4:3";
+ String webAppAddress = "1.2.3.4:4";
+
+ return SubClusterInfo.newInstance(subClusterId, amRMAddress,
+ clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
+ CLOCK.getTime(), "capability");
+ }
+
+ private void registerSubCluster(SubClusterId subClusterId)
+ throws YarnException {
+
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ }
+
+ public void registerSubClusters(int numSubClusters) throws YarnException {
+
+ for (int i = 0; i < numSubClusters; i++) {
+ registerSubCluster(SubClusterId.newInstance(SC_PREFIX + i));
+ }
+ }
+
+ private void addApplicationHomeSC(ApplicationId appId,
+ SubClusterId subClusterId) throws YarnException {
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc);
+ stateStore.addApplicationHomeSubCluster(request);
+ }
+
+ public void addAppsHomeSC(long clusterTs, int numApps) throws YarnException {
+ for (int i = 0; i < numApps; i++) {
+ addApplicationHomeSC(ApplicationId.newInstance(clusterTs, i),
+ SubClusterId.newInstance(SC_PREFIX + i));
+ }
+ }
+
+ private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
+ String policyType) {
+ return SubClusterPolicyConfiguration.newInstance(queueName, policyType,
+ ByteBuffer.allocate(1));
+ }
+
+ private void setPolicyConf(String queue, String policyType)
+ throws YarnException {
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf(queue, policyType));
+ stateStore.setPolicyConfiguration(request);
+ }
+
+ public void addPolicyConfigs(int numQueues) throws YarnException {
+
+ for (int i = 0; i < numQueues; i++) {
+ setPolicyConf(Q_PREFIX + i, POLICY_PREFIX + i);
+ }
+ }
+
+ public SubClusterInfo querySubClusterInfo(SubClusterId subClusterId)
+ throws YarnException {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ return stateStore.getSubCluster(request).getSubClusterInfo();
+ }
+
+ public SubClusterId queryApplicationHomeSC(ApplicationId appId)
+ throws YarnException {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ GetApplicationHomeSubClusterResponse response =
+ stateStore.getApplicationHomeSubCluster(request);
+
+ return response.getApplicationHomeSubCluster().getHomeSubCluster();
+ }
+
+ public SubClusterPolicyConfiguration queryPolicyConfiguration(String queue)
+ throws YarnException {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue);
+
+ GetSubClusterPolicyConfigurationResponse result =
+ stateStore.getPolicyConfiguration(request);
+ return result.getPolicyConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea752f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
new file mode 100644
index 0000000..53f4f84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Unit tests for FederationStateStoreFacade.
+ */
+@RunWith(Parameterized.class)
+public class TestFederationStateStoreFacade {
+
+ @Parameters
+ public static Collection<Boolean[]> getParameters() {
+ return Arrays
+ .asList(new Boolean[][] {{Boolean.FALSE }, {Boolean.TRUE } });
+ }
+
+ private final long clusterTs = System.currentTimeMillis();
+ private final int numSubClusters = 3;
+ private final int numApps = 5;
+ private final int numQueues = 2;
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private FederationStateStoreTestUtil stateStoreTestUtil;
+ private FederationStateStoreFacade facade =
+ FederationStateStoreFacade.getInstance();
+
+ public TestFederationStateStoreFacade(Boolean isCachingEnabled) {
+ conf = new Configuration();
+ if (!(isCachingEnabled.booleanValue())) {
+ conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
+ }
+ }
+
+ @Before
+ public void setUp() throws IOException, YarnException {
+ stateStore = new MemoryFederationStateStore();
+ stateStore.init(conf);
+ facade.reinitialize(stateStore, conf);
+ // hydrate the store
+ stateStoreTestUtil = new FederationStateStoreTestUtil(stateStore);
+ stateStoreTestUtil.registerSubClusters(numSubClusters);
+ stateStoreTestUtil.addAppsHomeSC(clusterTs, numApps);
+ stateStoreTestUtil.addPolicyConfigs(numQueues);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stateStore.close();
+ stateStore = null;
+ }
+
+ @Test
+ public void testGetSubCluster() throws YarnException {
+ for (int i = 0; i < numSubClusters; i++) {
+ SubClusterId subClusterId =
+ SubClusterId.newInstance(FederationStateStoreTestUtil.SC_PREFIX + i);
+ Assert.assertEquals(stateStoreTestUtil.querySubClusterInfo(subClusterId),
+ facade.getSubCluster(subClusterId));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterFlushCache() throws YarnException {
+ for (int i = 0; i < numSubClusters; i++) {
+ SubClusterId subClusterId =
+ SubClusterId.newInstance(FederationStateStoreTestUtil.SC_PREFIX + i);
+ Assert.assertEquals(stateStoreTestUtil.querySubClusterInfo(subClusterId),
+ facade.getSubCluster(subClusterId, true));
+ }
+ }
+
+ @Test
+ public void testGetSubClusters() throws YarnException {
+ Map<SubClusterId, SubClusterInfo> subClusters =
+ facade.getSubClusters(false);
+ for (SubClusterId subClusterId : subClusters.keySet()) {
+ Assert.assertEquals(stateStoreTestUtil.querySubClusterInfo(subClusterId),
+ subClusters.get(subClusterId));
+ }
+ }
+
+ @Test
+ public void testGetPolicyConfiguration() throws YarnException {
+ for (int i = 0; i < numQueues; i++) {
+ String queue = FederationStateStoreTestUtil.Q_PREFIX + i;
+ Assert.assertEquals(stateStoreTestUtil.queryPolicyConfiguration(queue),
+ facade.getPolicyConfiguration(queue));
+ }
+ }
+
+ @Test
+ public void testGetPoliciesConfigurations() throws YarnException {
+ Map<String, SubClusterPolicyConfiguration> queuePolicies =
+ facade.getPoliciesConfigurations();
+ for (String queue : queuePolicies.keySet()) {
+ Assert.assertEquals(stateStoreTestUtil.queryPolicyConfiguration(queue),
+ queuePolicies.get(queue));
+ }
+ }
+
+ @Test
+ public void testGetHomeSubClusterForApp() throws YarnException {
+ for (int i = 0; i < numApps; i++) {
+ ApplicationId appId = ApplicationId.newInstance(clusterTs, i);
+ Assert.assertEquals(stateStoreTestUtil.queryApplicationHomeSC(appId),
+ facade.getApplicationHomeSubCluster(appId));
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-3662. Federation Membership
State Store internal APIs.
Posted by su...@apache.org.
YARN-3662. Federation Membership State Store internal APIs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00b32dff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00b32dff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00b32dff
Branch: refs/heads/YARN-2915
Commit: 00b32dff4d56b3d867fa138e382c7a7a5985adbe
Parents: 4aca205
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Jul 29 16:53:40 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Feb 14 16:01:14 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn-server-common/pom.xml | 8 +
.../store/FederationMembershipStateStore.java | 126 +++++++++
.../server/federation/store/package-info.java | 17 ++
.../store/records/GetSubClusterInfoRequest.java | 62 +++++
.../records/GetSubClusterInfoResponse.java | 62 +++++
.../records/GetSubClustersInfoRequest.java | 66 +++++
.../records/GetSubClustersInfoResponse.java | 66 +++++
.../records/SubClusterDeregisterRequest.java | 89 +++++++
.../records/SubClusterDeregisterResponse.java | 42 +++
.../records/SubClusterHeartbeatRequest.java | 149 +++++++++++
.../records/SubClusterHeartbeatResponse.java | 45 ++++
.../federation/store/records/SubClusterId.java | 100 +++++++
.../store/records/SubClusterInfo.java | 263 ++++++++++++++++++
.../records/SubClusterRegisterRequest.java | 74 +++++
.../records/SubClusterRegisterResponse.java | 44 +++
.../store/records/SubClusterState.java | 60 +++++
.../impl/pb/GetSubClusterInfoRequestPBImpl.java | 125 +++++++++
.../pb/GetSubClusterInfoResponsePBImpl.java | 134 ++++++++++
.../pb/GetSubClustersInfoRequestPBImpl.java | 108 ++++++++
.../pb/GetSubClustersInfoResponsePBImpl.java | 184 +++++++++++++
.../pb/SubClusterDeregisterRequestPBImpl.java | 156 +++++++++++
.../pb/SubClusterDeregisterResponsePBImpl.java | 77 ++++++
.../pb/SubClusterHeartbeatRequestPBImpl.java | 192 +++++++++++++
.../pb/SubClusterHeartbeatResponsePBImpl.java | 77 ++++++
.../records/impl/pb/SubClusterIdPBImpl.java | 75 ++++++
.../records/impl/pb/SubClusterInfoPBImpl.java | 267 +++++++++++++++++++
.../pb/SubClusterRegisterRequestPBImpl.java | 134 ++++++++++
.../pb/SubClusterRegisterResponsePBImpl.java | 77 ++++++
.../store/records/impl/pb/package-info.java | 17 ++
.../federation/store/records/package-info.java | 17 ++
.../proto/yarn_server_federation_protos.proto | 93 +++++++
.../records/TestFederationProtocolRecords.java | 133 +++++++++
32 files changed, 3139 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index fc23af8..9cc3cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -58,6 +58,13 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -146,6 +153,7 @@
<include>yarn_server_common_protos.proto</include>
<include>yarn_server_common_service_protos.proto</include>
<include>yarn_server_common_service_protos.proto</include>
+ <include>yarn_server_federation_protos.proto</include>
<include>ResourceTracker.proto</include>
<include>SCMUploader.proto</include>
<include>collectornodemanager_protocol.proto</include>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
new file mode 100644
index 0000000..378eadc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+
+/**
+ * FederationMembershipStateStore maintains the state of all
+ * <em>subcluster(s)</em> as encapsulated by {@code SubClusterInfo} for all the
+ * subcluster(s) that are participating in federation.
+ */
+@Private
+@Unstable
+public interface FederationMembershipStateStore {
+
+ /**
+ * Get the {@link Version} of the underlying federation membership state
+ * store.
+ *
+ * @return the {@link Version} of the underlying federation membership state
+ * store
+ */
+ Version getMembershipStateStoreVersion();
+
+ /**
+ * Register a <em>subcluster</em> by publishing capabilities as represented by
+ * {@code SubClusterInfo} to indicate participation in federation. This is
+ * typically done during initialization or restart/failover of the
+ * subcluster's <code>ResourceManager</code>. Upon successful registration, an
+ * identifier for the <em>subcluster</em> which is unique across the federated
+ * cluster is returned. The identifier is static, i.e. preserved across
+ * restarts and failover.
+ *
+ * @param registerSubClusterRequest the capabilities of the subcluster that
+ * wants to participate in federation. The subcluster id is also
+ * specified in case registration is triggered by restart/failover
+ * @return response empty on successfully if registration was successful
+ * @throws YarnException if the request is invalid/fails
+ */
+ SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest registerSubClusterRequest) throws YarnException;
+
+ /**
+ * Deregister a <em>subcluster</em> identified by {@code SubClusterId} to
+ * change state in federation. This can be done to mark the sub cluster lost,
+ * deregistered, or decommissioned.
+ *
+ * @param subClusterDeregisterRequest - the request to deregister the
+ * sub-cluster from federation.
+ * @return response empty on successfully deregistering the subcluster state
+ * @throws YarnException if the request is invalid/fails
+ */
+ SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest subClusterDeregisterRequest)
+ throws YarnException;
+
+ /**
+ * Periodic heartbeat from a <code>ResourceManager</code> participating in
+ * federation to indicate liveliness. The heartbeat publishes the current
+ * capabilities as represented by {@code SubClusterInfo} of the subcluster.
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ *
+ * @param subClusterHeartbeatRequest the capabilities of the subcluster that
+ * wants to keep alive its participation in federation
+ * @return response currently empty on if heartbeat was successfully processed
+ * @throws YarnException if the request is invalid/fails
+ */
+ SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest subClusterHeartbeatRequest)
+ throws YarnException;
+
+ /**
+ * Get the membership information of <em>subcluster</em> as identified by
+ * {@code SubClusterId}. The membership information includes the cluster
+ * endpoint and current capabilities as represented by {@code SubClusterInfo}.
+ *
+ * @param subClusterRequest the subcluster whose information is required
+ * @return the {@code SubClusterInfo}
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest subClusterRequest) throws YarnException;
+
+ /**
+ * Get the membership information of all the <em>subclusters</em> that are
+ * currently participating in federation. The membership information includes
+ * the cluster endpoint and current capabilities as represented by
+ * {@code SubClusterInfo}.
+ *
+ * @param subClustersRequest request for sub-clusters information
+ * @return a map of {@code SubClusterInfo} keyed by the {@code SubClusterId}
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest subClustersRequest) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java
new file mode 100644
index 0000000..33179e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java
new file mode 100644
index 0000000..656dea9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain information about a sub-cluster identified by its
+ * {@link SubClusterId}.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterInfoRequest {
+
+ @Private
+ @Unstable
+ public static GetSubClusterInfoRequest newInstance(
+ SubClusterId subClusterId) {
+ GetSubClusterInfoRequest subClusterRequest =
+ Records.newRecord(GetSubClusterInfoRequest.class);
+ subClusterRequest.setSubClusterId(subClusterId);
+ return subClusterRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @param subClusterId the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java
new file mode 100644
index 0000000..f7bc74d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Response to a query with {@link SubClusterInfo} about a sub-cluster.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterInfoResponse {
+
+ @Private
+ @Unstable
+ public static GetSubClusterInfoResponse newInstance(
+ SubClusterInfo subClusterInfo) {
+ GetSubClusterInfoResponse registerSubClusterRequest =
+ Records.newRecord(GetSubClusterInfoResponse.class);
+ registerSubClusterRequest.setSubClusterInfo(subClusterInfo);
+ return registerSubClusterRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @return the information pertaining to the sub-cluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterInfo getSubClusterInfo();
+
+ /**
+ * Set the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @param subClusterInfo the information pertaining to the sub-cluster
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterInfo(SubClusterInfo subClusterInfo);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
new file mode 100644
index 0000000..3264d81
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain information about all sub-clusters that are
+ * participating in federation.
+ */
+@Private
+@Unstable
+public abstract class GetSubClustersInfoRequest {
+
+ @Public
+ @Unstable
+ public static GetSubClustersInfoRequest newInstance(
+ boolean filterInactiveSubClusters) {
+ GetSubClustersInfoRequest request =
+ Records.newRecord(GetSubClustersInfoRequest.class);
+ request.setFilterInactiveSubClusters(filterInactiveSubClusters);
+ return request;
+ }
+
+ /**
+ * Get the flag that indicates whether only active sub-clusters should be
+ * returned.
+ *
+ * @return whether to filter out inactive sub-clusters
+ */
+ @Public
+ @Unstable
+ public abstract boolean getFilterInactiveSubClusters();
+
+ /**
+ * Set the flag that indicates whether only active sub-clusters should be
+ * returned.
+ *
+ * @param filterInactiveSubClusters whether to filter out inactive
+ * sub-clusters
+ */
+ @Public
+ @Unstable
+ public abstract void setFilterInactiveSubClusters(
+ boolean filterInactiveSubClusters);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
new file mode 100644
index 0000000..bcf75ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Response to a query with list of {@link SubClusterInfo} about all
+ * sub-clusters that are currently participating in Federation.
+ */
+@Private
+@Unstable
+public abstract class GetSubClustersInfoResponse {
+
+ @Public
+ @Unstable
+ public static GetSubClustersInfoResponse newInstance(
+ List<SubClusterInfo> subClusters) {
+ GetSubClustersInfoResponse subClusterInfos =
+ Records.newRecord(GetSubClustersInfoResponse.class);
+ subClusterInfos.setSubClusters(subClusters);
+ return subClusterInfos;
+ }
+
+ /**
+ * Get the list of {@link SubClusterInfo} representing the information about
+ * all sub-clusters that are currently participating in Federation.
+ *
+ * @return the list of {@link SubClusterInfo}
+ */
+ @Public
+ @Unstable
+ public abstract List<SubClusterInfo> getSubClusters();
+
+ /**
+ * Set the list of {@link SubClusterInfo} representing the information about
+ * all sub-clusters that are currently participating in Federation.
+ *
+ * @param subClusters the list of {@link SubClusterInfo}
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusters(List<SubClusterInfo> subClusters);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java
new file mode 100644
index 0000000..50a50a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent to set the state of a subcluster to either
+ * SC_DECOMMISSIONED, SC_LOST, or SC_DEREGISTERED.
+ *
+ * <p>
+ * The update includes details such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>{@link SubClusterState}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterDeregisterRequest {
+
+ @Private
+ @Unstable
+ public static SubClusterDeregisterRequest newInstance(
+ SubClusterId subClusterId, SubClusterState subClusterState) {
+ SubClusterDeregisterRequest registerRequest =
+ Records.newRecord(SubClusterDeregisterRequest.class);
+ registerRequest.setSubClusterId(subClusterId);
+ registerRequest.setState(subClusterState);
+ return registerRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @param subClusterId the subcluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+
+ /**
+ * Get the {@link SubClusterState} of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterState getState();
+
+ /**
+ * Set the {@link SubClusterState} of the subcluster.
+ *
+ * @param state the state of the subCluster
+ */
+ @Private
+ @Unstable
+ public abstract void setState(SubClusterState state);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java
new file mode 100644
index 0000000..74fe994
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SubClusterDeregisterResponse contains the answer from the {@code
+ * FederationMembershipStateStore} to a request to deregister the sub cluster.
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ */
+@Private
+@Unstable
+public abstract class SubClusterDeregisterResponse {
+
+ @Private
+ @Unstable
+ public static SubClusterDeregisterResponse newInstance() {
+ SubClusterDeregisterResponse response =
+ Records.newRecord(SubClusterDeregisterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java
new file mode 100644
index 0000000..3a07c18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterHeartbeatRequest is a report of the runtime information of the
+ * subcluster that is participating in federation.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>The URL of the subcluster</li>
+ * <li>The timestamp representing the last start time of the subCluster</li>
+ * <li>{@code FederationsubClusterState}</li>
+ * <li>The current capacity and utilization of the subCluster</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterHeartbeatRequest {
+
+ @Private
+ @Unstable
+ public static SubClusterHeartbeatRequest newInstance(
+ SubClusterId subClusterId, SubClusterState state, String capability) {
+ return newInstance(subClusterId, 0, state, capability);
+ }
+
+ @Private
+ @Unstable
+ public static SubClusterHeartbeatRequest newInstance(
+ SubClusterId subClusterId, long lastHeartBeat, SubClusterState state,
+ String capability) {
+ SubClusterHeartbeatRequest subClusterHeartbeatRequest =
+ Records.newRecord(SubClusterHeartbeatRequest.class);
+ subClusterHeartbeatRequest.setSubClusterId(subClusterId);
+ subClusterHeartbeatRequest.setLastHeartBeat(lastHeartBeat);
+ subClusterHeartbeatRequest.setState(state);
+ subClusterHeartbeatRequest.setCapability(capability);
+ return subClusterHeartbeatRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subCluster.
+ *
+ * @param subClusterId the subCluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+
+ /**
+ * Get the last heart beat time of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract long getLastHeartBeat();
+
+ /**
+ * Set the last heartbeat time of the subcluster.
+ *
+ * @param time the last heartbeat time of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setLastHeartBeat(long time);
+
+ /**
+ * Get the {@link SubClusterState} of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterState getState();
+
+ /**
+ * Set the {@link SubClusterState} of the subcluster.
+ *
+ * @param state the state of the subCluster
+ */
+ @Private
+ @Unstable
+ public abstract void setState(SubClusterState state);
+
+ /**
+ * Get the current capacity and utilization of the subcluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @return the current capacity and utilization of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract String getCapability();
+
+ /**
+ * Set the current capacity and utilization of the subCluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @param capability the current capacity and utilization of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setCapability(String capability);
+
+ @Override
+ public String toString() {
+ return "SubClusterHeartbeatRequest [getSubClusterId() = "
+ + getSubClusterId() + ", getState() = " + getState()
+ + ", getLastHeartBeat = " + getLastHeartBeat() + ", getCapability() = "
+ + getCapability() + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java
new file mode 100644
index 0000000..0b7fd8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SubClusterHeartbeatResponse contains the response from the {@code
+ * FederationMembershipStateStore} to a periodic heartbeat to indicate
+ * liveliness from a <code>ResourceManager</code> participating in federation.
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ * <p>
+ * NOTE: This can be extended to push down policies in future
+ */
+@Private
+@Unstable
+public abstract class SubClusterHeartbeatResponse {
+
+ @Private
+ @Unstable
+ public static SubClusterHeartbeatResponse newInstance() {
+ SubClusterHeartbeatResponse response =
+ Records.newRecord(SubClusterHeartbeatResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
new file mode 100644
index 0000000..fec967d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterId represents the <em>globally unique</em> identifier for a
+ * subcluster that is participating in federation.
+ *
+ * <p>
+ * The globally unique nature of the identifier is obtained from the
+ * <code>FederationMembershipStateStore</code> on initialization.
+ */
+@Private
+@Unstable
+public abstract class SubClusterId implements Comparable<SubClusterId> {
+
+ @Private
+ @Unstable
+ public static SubClusterId newInstance(String subClusterId) {
+ SubClusterId id = Records.newRecord(SubClusterId.class);
+ id.setId(subClusterId);
+ return id;
+ }
+
+ /**
+ * Get the string identifier of the <em>subcluster</em> which is unique across
+ * the federated cluster. The identifier is static, i.e. preserved across
+ * restarts and failover.
+ *
+ * @return unique identifier of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract String getId();
+
+ /**
+ * Set the string identifier of the <em>subcluster</em> which is unique across
+ * the federated cluster. The identifier is static, i.e. preserved across
+ * restarts and failover.
+ *
+ * @param subClusterId unique identifier of the subcluster
+ */
+ @Private
+ @Unstable
+ protected abstract void setId(String subClusterId);
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ SubClusterId other = (SubClusterId) obj;
+ return this.getId().equals(other.getId());
+ }
+
+ @Override
+ public int hashCode() {
+ return getId().hashCode();
+ }
+
+ @Override
+ public int compareTo(SubClusterId other) {
+ return getId().compareTo(other.getId());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getId());
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
new file mode 100644
index 0000000..f13c8f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterInfo is a report of the runtime information of the subcluster that
+ * is participating in federation.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>The URL of the subcluster</li>
+ * <li>The timestamp representing the last start time of the subCluster</li>
+ * <li>{@code FederationsubClusterState}</li>
+ * <li>The current capacity and utilization of the subCluster</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterInfo {
+
+ @Private
+ @Unstable
+ public static SubClusterInfo newInstance(SubClusterId subClusterId,
+ String amRMServiceAddress, String clientRMServiceAddress,
+ String rmAdminServiceAddress, String rmWebServiceAddress,
+ SubClusterState state, long lastStartTime, String capability) {
+ return newInstance(subClusterId, amRMServiceAddress, clientRMServiceAddress,
+ rmAdminServiceAddress, rmWebServiceAddress, 0, state, lastStartTime,
+ capability);
+ }
+
+ @Private
+ @Unstable
+ public static SubClusterInfo newInstance(SubClusterId subClusterId,
+ String amRMServiceAddress, String clientRMServiceAddress,
+ String rmAdminServiceAddress, String rmWebServiceAddress,
+ long lastHeartBeat, SubClusterState state, long lastStartTime,
+ String capability) {
+ SubClusterInfo subClusterInfo = Records.newRecord(SubClusterInfo.class);
+ subClusterInfo.setSubClusterId(subClusterId);
+ subClusterInfo.setAMRMServiceAddress(amRMServiceAddress);
+ subClusterInfo.setClientRMServiceAddress(clientRMServiceAddress);
+ subClusterInfo.setRMAdminServiceAddress(rmAdminServiceAddress);
+ subClusterInfo.setRMWebServiceAddress(rmWebServiceAddress);
+ subClusterInfo.setLastHeartBeat(lastHeartBeat);
+ subClusterInfo.setState(state);
+ subClusterInfo.setLastStartTime(lastStartTime);
+ subClusterInfo.setCapability(capability);
+ return subClusterInfo;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subCluster.
+ *
+ * @param subClusterId the subCluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+
+ /**
+ * Get the URL of the AM-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @return the URL of the AM-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>
+ */
+ @Public
+ @Unstable
+ public abstract String getAMRMServiceAddress();
+
+ /**
+ * Set the URL of the AM-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @param amRMServiceAddress the URL of the AM-RM service endpoint of the
+ * subcluster <code>ResourceManager</code>
+ */
+ @Private
+ @Unstable
+ public abstract void setAMRMServiceAddress(String amRMServiceAddress);
+
+ /**
+ * Get the URL of the client-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @return the URL of the client-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>
+ */
+ @Public
+ @Unstable
+ public abstract String getClientRMServiceAddress();
+
+ /**
+ * Set the URL of the client-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @param clientRMServiceAddress the URL of the client-RM service endpoint of
+ * the subCluster <code>ResourceManager</code>
+ */
+ @Private
+ @Unstable
+ public abstract void setClientRMServiceAddress(String clientRMServiceAddress);
+
+ /**
+ * Get the URL of the <code>ResourceManager</code> administration service.
+ *
+ * @return the URL of the <code>ResourceManager</code> administration service
+ */
+ @Public
+ @Unstable
+ public abstract String getRMAdminServiceAddress();
+
+ /**
+ * Set the URL of the <code>ResourceManager</code> administration service.
+ *
+ * @param rmAdminServiceAddress the URL of the <code>ResourceManager</code>
+ * administration service.
+ */
+ @Private
+ @Unstable
+ public abstract void setRMAdminServiceAddress(String rmAdminServiceAddress);
+
+ /**
+ * Get the URL of the <code>ResourceManager</code> web application interface.
+ *
+ * @return the URL of the <code>ResourceManager</code> web application
+ * interface.
+ */
+ @Public
+ @Unstable
+ public abstract String getRMWebServiceAddress();
+
+ /**
+ * Set the URL of the <code>ResourceManager</code> web application interface.
+ *
+ * @param rmWebServiceAddress the URL of the <code>ResourceManager</code> web
+ * application interface.
+ */
+ @Private
+ @Unstable
+ public abstract void setRMWebServiceAddress(String rmWebServiceAddress);
+
+ /**
+ * Get the last heart beat time of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract long getLastHeartBeat();
+
+ /**
+ * Set the last heartbeat time of the subcluster.
+ *
+ * @param time the last heartbeat time of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setLastHeartBeat(long time);
+
+ /**
+ * Get the {@link SubClusterState} of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterState getState();
+
+ /**
+ * Set the {@link SubClusterState} of the subcluster.
+ *
+ * @param state the state of the subCluster
+ */
+ @Private
+ @Unstable
+ public abstract void setState(SubClusterState state);
+
+ /**
+ * Get the timestamp representing the last start time of the subcluster.
+ *
+ * @return the timestamp representing the last start time of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract long getLastStartTime();
+
+ /**
+ * Set the timestamp representing the last start time of the subcluster.
+ *
+ * @param lastStartTime the timestamp representing the last start time of the
+ * subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setLastStartTime(long lastStartTime);
+
+ /**
+ * Get the current capacity and utilization of the subcluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @return the current capacity and utilization of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract String getCapability();
+
+ /**
+ * Set the current capacity and utilization of the subCluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @param capability the current capacity and utilization of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setCapability(String capability);
+
+ @Override
+ public String toString() {
+ return "SubClusterInfo [getSubClusterId() = " + getSubClusterId()
+ + ", getAMRMServiceAddress() = " + getAMRMServiceAddress()
+ + ", getClientRMServiceAddress() = " + getClientRMServiceAddress()
+ + ", getRMAdminServiceAddress() = " + getRMAdminServiceAddress()
+ + ", getRMWebServiceAddress() = " + getRMWebServiceAddress()
+ + ", getState() = " + getState() + ", getLastStartTime() = "
+ + getLastStartTime() + ", getCapability() = " + getCapability() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java
new file mode 100644
index 0000000..8864fe3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterRegisterRequest is a request by a sub-cluster
+ * {@code ResourceManager} to participate in federation.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>The URL of the subcluster</li>
+ * <li>The timestamp representing the last start time of the subCluster</li>
+ * <li>{@code FederationsubClusterState}</li>
+ * <li>The current capacity and utilization of the subCluster</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterRegisterRequest {
+
+ @Private
+ @Unstable
+ public static SubClusterRegisterRequest newInstance(
+ SubClusterInfo subClusterInfo) {
+ SubClusterRegisterRequest registerSubClusterRequest =
+ Records.newRecord(SubClusterRegisterRequest.class);
+ registerSubClusterRequest.setSubClusterInfo(subClusterInfo);
+ return registerSubClusterRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @return the information pertaining to the sub-cluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterInfo getSubClusterInfo();
+
+ /**
+ * Set the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @param subClusterInfo the information pertaining to the sub-cluster
+ */
+ @Public
+ @Unstable
+ public abstract void setSubClusterInfo(SubClusterInfo subClusterInfo);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java
new file mode 100644
index 0000000..060a857
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SubClusterRegisterResponse contains the response from the {@code
+ * FederationMembershipStateStore} to a registration request from a
+ * <code>ResourceManager</code> to participate in federation.
+ *
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ */
+@Private
+@Unstable
+public abstract class SubClusterRegisterResponse {
+
+ @Private
+ @Unstable
+ public static SubClusterRegisterResponse newInstance() {
+ SubClusterRegisterResponse response =
+ Records.newRecord(SubClusterRegisterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
new file mode 100644
index 0000000..22cec99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p>
+ * State of a <code>SubCluster</code>.
+ * </p>
+ */
+@Private
+@Unstable
+public enum SubClusterState {
+ /** Newly registered subcluster, before the first heartbeat. */
+ SC_NEW,
+
+ /** Subcluster is registered and the RM sent a heartbeat recently. */
+ SC_RUNNING,
+
+ /** Subcluster is unhealthy. */
+ SC_UNHEALTHY,
+
+ /** Subcluster is in the process of being out of service. */
+ SC_DECOMMISSIONING,
+
+ /** Subcluster is out of service. */
+ SC_DECOMMISSIONED,
+
+ /** RM has not sent a heartbeat for some configured time threshold. */
+ SC_LOST,
+
+ /** Subcluster has unregistered. */
+ SC_UNREGISTERED;
+
+ public boolean isUnusable() {
+ return (this != SC_RUNNING && this != SC_NEW);
+ }
+
+ public boolean isFinal() {
+ return (this == SC_UNREGISTERED || this == SC_DECOMMISSIONED
+ || this == SC_LOST);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java
new file mode 100644
index 0000000..c61c419
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClusterInfoRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClusterInfoRequestPBImpl extends GetSubClusterInfoRequest {
+
+ private GetSubClusterInfoRequestProto proto =
+ GetSubClusterInfoRequestProto.getDefaultInstance();
+ private GetSubClusterInfoRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetSubClusterInfoRequestPBImpl() {
+ builder = GetSubClusterInfoRequestProto.newBuilder();
+ }
+
+ public GetSubClusterInfoRequestPBImpl(GetSubClusterInfoRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterInfoRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterInfoRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ GetSubClusterInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getSubClusterId());
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ return;
+ }
+ builder.setSubClusterId(convertToProtoFormat(subClusterId));
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+ return new SubClusterIdPBImpl(sc);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+ return ((SubClusterIdPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
new file mode 100644
index 0000000..d0bcc33
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClusterInfoResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClusterInfoResponsePBImpl extends GetSubClusterInfoResponse {
+
+ private GetSubClusterInfoResponseProto proto =
+ GetSubClusterInfoResponseProto.getDefaultInstance();
+ private GetSubClusterInfoResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterInfo subClusterInfo = null;
+
+ public GetSubClusterInfoResponsePBImpl() {
+ builder = GetSubClusterInfoResponseProto.newBuilder();
+ }
+
+ public GetSubClusterInfoResponsePBImpl(GetSubClusterInfoResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterInfoResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterInfoResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterInfo != null) {
+ builder.setSubClusterInfo(convertToProtoFormat(this.subClusterInfo));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterInfo getSubClusterInfo() {
+ GetSubClusterInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterInfo != null) {
+ return this.subClusterInfo;
+ }
+ if (!p.hasSubClusterInfo()) {
+ return null;
+ }
+ this.subClusterInfo = convertFromProtoFormat(p.getSubClusterInfo());
+ return this.subClusterInfo;
+ }
+
+ @Override
+ public void setSubClusterInfo(SubClusterInfo subClusterInfo) {
+ maybeInitBuilder();
+ if (subClusterInfo == null) {
+ builder.clearSubClusterInfo();
+ }
+ this.subClusterInfo = subClusterInfo;
+ }
+
+ private SubClusterInfo convertFromProtoFormat(
+ SubClusterInfoProto clusterInfo) {
+ return new SubClusterInfoPBImpl(clusterInfo);
+ }
+
+ private SubClusterInfoProto convertToProtoFormat(SubClusterInfo clusterInfo) {
+ return ((SubClusterInfoPBImpl) clusterInfo).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java
new file mode 100644
index 0000000..2b848c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClustersInfoRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClustersInfoRequestPBImpl extends GetSubClustersInfoRequest {
+
+ private GetSubClustersInfoRequestProto proto =
+ GetSubClustersInfoRequestProto.getDefaultInstance();
+ private GetSubClustersInfoRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetSubClustersInfoRequestPBImpl() {
+ builder = GetSubClustersInfoRequestProto.newBuilder();
+ }
+
+ public GetSubClustersInfoRequestPBImpl(GetSubClustersInfoRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClustersInfoRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClustersInfoRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public boolean getFilterInactiveSubClusters() {
+ GetSubClustersInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getFilterInactiveSubclusters();
+ }
+
+ @Override
+ public void setFilterInactiveSubClusters(boolean filterInactiveSubClusters) {
+ maybeInitBuilder();
+ builder.setFilterInactiveSubclusters(filterInactiveSubClusters);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b32dff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
new file mode 100644
index 0000000..d39ef7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClustersInfoResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClustersInfoResponsePBImpl
+ extends GetSubClustersInfoResponse {
+
+ private GetSubClustersInfoResponseProto proto =
+ GetSubClustersInfoResponseProto.getDefaultInstance();
+ private GetSubClustersInfoResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List<SubClusterInfo> subClusterInfos;
+
+ public GetSubClustersInfoResponsePBImpl() {
+ builder = GetSubClustersInfoResponseProto.newBuilder();
+ }
+
+ public GetSubClustersInfoResponsePBImpl(
+ GetSubClustersInfoResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClustersInfoResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterInfos != null) {
+ addReservationResourcesToProto();
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClustersInfoResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public List<SubClusterInfo> getSubClusters() {
+ initSubClustersInfoList();
+ return subClusterInfos;
+ }
+
+ @Override
+ public void setSubClusters(List<SubClusterInfo> subClusters) {
+ if (subClusters == null) {
+ builder.clearSubClusterInfos();
+ return;
+ }
+ this.subClusterInfos = subClusters;
+ }
+
+ private void initSubClustersInfoList() {
+ if (this.subClusterInfos != null) {
+ return;
+ }
+ GetSubClustersInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<SubClusterInfoProto> subClusterInfosList = p.getSubClusterInfosList();
+ subClusterInfos = new ArrayList<SubClusterInfo>();
+
+ for (SubClusterInfoProto r : subClusterInfosList) {
+ subClusterInfos.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addReservationResourcesToProto() {
+ maybeInitBuilder();
+ builder.clearSubClusterInfos();
+ if (subClusterInfos == null) {
+ return;
+ }
+ Iterable<SubClusterInfoProto> iterable =
+ new Iterable<SubClusterInfoProto>() {
+ @Override
+ public Iterator<SubClusterInfoProto> iterator() {
+ return new Iterator<SubClusterInfoProto>() {
+
+ private Iterator<SubClusterInfo> iter =
+ subClusterInfos.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public SubClusterInfoProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllSubClusterInfos(iterable);
+ }
+
+ private SubClusterInfoProto convertToProtoFormat(SubClusterInfo r) {
+ return ((SubClusterInfoPBImpl) r).getProto();
+ }
+
+ private SubClusterInfoPBImpl convertFromProtoFormat(SubClusterInfoProto r) {
+ return new SubClusterInfoPBImpl(r);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org