You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/02/24 19:34:28 UTC
[01/50] [abbrv] hadoop git commit: YARN-6143. Fix incompatible issue
caused by YARN-3583. (Sunil G via wangda) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-2915 ace91d5cf -> d9bada3ee (forced update)
YARN-6143. Fix incompatible issue caused by YARN-3583. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a6ca75f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a6ca75f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a6ca75f
Branch: refs/heads/YARN-2915
Commit: 1a6ca75f3872587fb34c995a4b372b8cd6366d7d
Parents: cfcd527
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Feb 22 11:17:09 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Feb 22 11:17:09 2017 -0800
----------------------------------------------------------------------
.../hadoop/mapred/ResourceMgrDelegate.java | 6 +--
.../GetLabelsToNodesResponse.java | 6 +--
.../GetNodesToLabelsResponse.java | 7 ++-
.../AddToClusterNodeLabelsRequest.java | 11 ++---
..._server_resourcemanager_service_protos.proto | 5 +-
.../src/main/proto/yarn_protos.proto | 6 +--
.../src/main/proto/yarn_service_protos.proto | 5 +-
.../hadoop/yarn/client/api/YarnClient.java | 6 +--
.../yarn/client/api/impl/YarnClientImpl.java | 6 +--
.../yarn/client/api/impl/TestYarnClient.java | 49 ++++++++----------
.../pb/GetClusterNodeLabelsResponsePBImpl.java | 4 ++
.../impl/pb/GetLabelsToNodesResponsePBImpl.java | 20 +++-----
.../impl/pb/GetNodesToLabelsResponsePBImpl.java | 52 ++++++++------------
.../pb/AddToClusterNodeLabelsRequestPBImpl.java | 47 +++++++++---------
.../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +++----
.../server/resourcemanager/AdminService.java | 7 +--
.../server/resourcemanager/ClientRMService.java | 6 +--
.../resourcemanager/TestClientRMService.java | 42 +++++-----------
18 files changed, 136 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index c302553..62aa497 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -476,19 +476,19 @@ public class ResourceMgrDelegate extends YarnClient {
return client.listReservations(request);
}
@Override
- public Map<NodeId, Set<NodeLabel>> getNodeToLabels() throws YarnException,
+ public Map<NodeId, Set<String>> getNodeToLabels() throws YarnException,
IOException {
return client.getNodeToLabels();
}
@Override
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes() throws YarnException,
+ public Map<String, Set<NodeId>> getLabelsToNodes() throws YarnException,
IOException {
return client.getLabelsToNodes();
}
@Override
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes(Set<String> labels)
+ public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
throws YarnException, IOException {
return client.getLabelsToNodes(labels);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
index da2be28..ef0bf60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class GetLabelsToNodesResponse {
public static GetLabelsToNodesResponse newInstance(
- Map<NodeLabel, Set<NodeId>> map) {
+ Map<String, Set<NodeId>> map) {
GetLabelsToNodesResponse response =
Records.newRecord(GetLabelsToNodesResponse.class);
response.setLabelsToNodes(map);
@@ -38,9 +38,9 @@ public abstract class GetLabelsToNodesResponse {
@Public
@Evolving
- public abstract void setLabelsToNodes(Map<NodeLabel, Set<NodeId>> map);
+ public abstract void setLabelsToNodes(Map<String, Set<NodeId>> map);
@Public
@Evolving
- public abstract Map<NodeLabel, Set<NodeId>> getLabelsToNodes();
+ public abstract Map<String, Set<NodeId>> getLabelsToNodes();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
index 432485c..bcd5421 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
@@ -24,12 +24,11 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.util.Records;
public abstract class GetNodesToLabelsResponse {
public static GetNodesToLabelsResponse newInstance(
- Map<NodeId, Set<NodeLabel>> map) {
+ Map<NodeId, Set<String>> map) {
GetNodesToLabelsResponse response =
Records.newRecord(GetNodesToLabelsResponse.class);
response.setNodeToLabels(map);
@@ -38,9 +37,9 @@ public abstract class GetNodesToLabelsResponse {
@Public
@Evolving
- public abstract void setNodeToLabels(Map<NodeId, Set<NodeLabel>> map);
+ public abstract void setNodeToLabels(Map<NodeId, Set<String>> map);
@Public
@Evolving
- public abstract Map<NodeId, Set<NodeLabel>> getNodeToLabels();
+ public abstract Map<NodeId, Set<String>> getNodeToLabels();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java
index c1ea07d..f2ac395 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java
@@ -30,17 +30,16 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class AddToClusterNodeLabelsRequest {
@Public
@Unstable
- public static AddToClusterNodeLabelsRequest newInstance(
- List<NodeLabel> NodeLabels) {
- AddToClusterNodeLabelsRequest request =
- Records.newRecord(AddToClusterNodeLabelsRequest.class);
- request.setNodeLabels(NodeLabels);
+ public static AddToClusterNodeLabelsRequest newInstance(List<NodeLabel> nodeLabels) {
+ AddToClusterNodeLabelsRequest request = Records
+ .newRecord(AddToClusterNodeLabelsRequest.class);
+ request.setNodeLabels(nodeLabels);
return request;
}
@Public
@Unstable
- public abstract void setNodeLabels(List<NodeLabel> NodeLabels);
+ public abstract void setNodeLabels(List<NodeLabel> nodeLabels);
@Public
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
index 16d8097..e8c92d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -84,7 +84,8 @@ message RefreshNodesResourcesResponseProto {
}
message AddToClusterNodeLabelsRequestProto {
- repeated NodeLabelProto nodeLabels = 1;
+ repeated string deprecatedNodeLabels = 1;
+ repeated NodeLabelProto nodeLabels = 2;
}
message AddToClusterNodeLabelsResponseProto {
@@ -98,7 +99,7 @@ message RemoveFromClusterNodeLabelsResponseProto {
}
message ReplaceLabelsOnNodeRequestProto {
- repeated NodeIdToLabelsNameProto nodeToLabels = 1;
+ repeated NodeIdToLabelsProto nodeToLabels = 1;
optional bool failOnUnknownNodes = 2;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index a6dbf3c..3b26a5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -282,13 +282,13 @@ message NodeReportProto {
optional ResourceUtilizationProto node_utilization = 12;
}
-message NodeIdToLabelsInfoProto {
+message NodeIdToLabelsProto {
optional NodeIdProto nodeId = 1;
- repeated NodeLabelProto nodeLabels = 2;
+ repeated string nodeLabels = 2;
}
message LabelsToNodeIdsProto {
- optional NodeLabelProto nodeLabels = 1;
+ optional string nodeLabels = 1;
repeated NodeIdProto nodeId = 2;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index c6647c8..0e14896 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -234,7 +234,7 @@ message GetNodesToLabelsRequestProto {
}
message GetNodesToLabelsResponseProto {
- repeated NodeIdToLabelsInfoProto nodeToLabels = 1;
+ repeated NodeIdToLabelsProto nodeToLabels = 1;
}
message GetLabelsToNodesRequestProto {
@@ -249,7 +249,8 @@ message GetClusterNodeLabelsRequestProto {
}
message GetClusterNodeLabelsResponseProto {
- repeated NodeLabelProto nodeLabels = 1;
+ repeated string deprecatedNodeLabels = 1;
+ repeated NodeLabelProto nodeLabels = 2;
}
message UpdateApplicationPriorityRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 0bf1643..8c68a31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -767,7 +767,7 @@ public abstract class YarnClient extends AbstractService {
*/
@Public
@Unstable
- public abstract Map<NodeId, Set<NodeLabel>> getNodeToLabels()
+ public abstract Map<NodeId, Set<String>> getNodeToLabels()
throws YarnException, IOException;
/**
@@ -782,7 +782,7 @@ public abstract class YarnClient extends AbstractService {
*/
@Public
@Unstable
- public abstract Map<NodeLabel, Set<NodeId>> getLabelsToNodes()
+ public abstract Map<String, Set<NodeId>> getLabelsToNodes()
throws YarnException, IOException;
/**
@@ -798,7 +798,7 @@ public abstract class YarnClient extends AbstractService {
*/
@Public
@Unstable
- public abstract Map<NodeLabel, Set<NodeId>> getLabelsToNodes(
+ public abstract Map<String, Set<NodeId>> getLabelsToNodes(
Set<String> labels) throws YarnException, IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 23b128c..8865b52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -883,21 +883,21 @@ public class YarnClientImpl extends YarnClient {
}
@Override
- public Map<NodeId, Set<NodeLabel>> getNodeToLabels() throws YarnException,
+ public Map<NodeId, Set<String>> getNodeToLabels() throws YarnException,
IOException {
return rmClient.getNodeToLabels(GetNodesToLabelsRequest.newInstance())
.getNodeToLabels();
}
@Override
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes() throws YarnException,
+ public Map<String, Set<NodeId>> getLabelsToNodes() throws YarnException,
IOException {
return rmClient.getLabelsToNodes(GetLabelsToNodesRequest.newInstance())
.getLabelsToNodes();
}
@Override
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes(Set<String> labels)
+ public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
throws YarnException, IOException {
return rmClient.getLabelsToNodes(
GetLabelsToNodesRequest.newInstance(labels)).getLabelsToNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index c2c9665..9adc187 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -609,9 +609,9 @@ public class TestYarnClient {
client.start();
// Get labels to nodes mapping
- Map<NodeLabel, Set<NodeId>> expectedLabelsToNodes =
+ Map<String, Set<NodeId>> expectedLabelsToNodes =
((MockYarnClient)client).getLabelsToNodesMap();
- Map<NodeLabel, Set<NodeId>> labelsToNodes = client.getLabelsToNodes();
+ Map<String, Set<NodeId>> labelsToNodes = client.getLabelsToNodes();
Assert.assertEquals(labelsToNodes, expectedLabelsToNodes);
Assert.assertEquals(labelsToNodes.size(), 3);
@@ -635,18 +635,12 @@ public class TestYarnClient {
client.start();
// Get labels to nodes mapping
- Map<NodeId, Set<NodeLabel>> expectedNodesToLabels = ((MockYarnClient) client)
+ Map<NodeId, Set<String>> expectedNodesToLabels = ((MockYarnClient) client)
.getNodeToLabelsMap();
- Map<NodeId, Set<NodeLabel>> nodesToLabels = client.getNodeToLabels();
+ Map<NodeId, Set<String>> nodesToLabels = client.getNodeToLabels();
Assert.assertEquals(nodesToLabels, expectedNodesToLabels);
Assert.assertEquals(nodesToLabels.size(), 1);
- // Verify exclusivity
- Set<NodeLabel> labels = nodesToLabels.get(NodeId.newInstance("host", 0));
- for (NodeLabel label : labels) {
- Assert.assertFalse(label.isExclusive());
- }
-
client.stop();
client.close();
}
@@ -884,7 +878,7 @@ public class TestYarnClient {
}
@Override
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes()
+ public Map<String, Set<NodeId>> getLabelsToNodes()
throws YarnException, IOException {
when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn(
getLabelsToNodesMap());
@@ -892,48 +886,45 @@ public class TestYarnClient {
}
@Override
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes(Set<String> labels)
+ public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
throws YarnException, IOException {
when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn(
getLabelsToNodesMap(labels));
return super.getLabelsToNodes(labels);
}
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodesMap() {
- Map<NodeLabel, Set<NodeId>> map = new HashMap<NodeLabel, Set<NodeId>>();
+ public Map<String, Set<NodeId>> getLabelsToNodesMap() {
+ Map<String, Set<NodeId>> map = new HashMap<String, Set<NodeId>>();
Set<NodeId> setNodeIds =
new HashSet<NodeId>(Arrays.asList(
NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0)));
- map.put(NodeLabel.newInstance("x"), setNodeIds);
- map.put(NodeLabel.newInstance("y"), setNodeIds);
- map.put(NodeLabel.newInstance("z"), setNodeIds);
+ map.put("x", setNodeIds);
+ map.put("y", setNodeIds);
+ map.put("z", setNodeIds);
return map;
}
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodesMap(Set<String> labels) {
- Map<NodeLabel, Set<NodeId>> map = new HashMap<NodeLabel, Set<NodeId>>();
- Set<NodeId> setNodeIds =
- new HashSet<NodeId>(Arrays.asList(
+ public Map<String, Set<NodeId>> getLabelsToNodesMap(Set<String> labels) {
+ Map<String, Set<NodeId>> map = new HashMap<String, Set<NodeId>>();
+ Set<NodeId> setNodeIds = new HashSet<NodeId>(Arrays.asList(
NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0)));
- for(String label : labels) {
- map.put(NodeLabel.newInstance(label), setNodeIds);
+ for (String label : labels) {
+ map.put(label, setNodeIds);
}
return map;
}
@Override
- public Map<NodeId, Set<NodeLabel>> getNodeToLabels() throws YarnException,
+ public Map<NodeId, Set<String>> getNodeToLabels() throws YarnException,
IOException {
when(mockNodeToLabelsResponse.getNodeToLabels()).thenReturn(
getNodeToLabelsMap());
return super.getNodeToLabels();
}
- public Map<NodeId, Set<NodeLabel>> getNodeToLabelsMap() {
- Map<NodeId, Set<NodeLabel>> map = new HashMap<NodeId, Set<NodeLabel>>();
- Set<NodeLabel> setNodeLabels = new HashSet<NodeLabel>(Arrays.asList(
- NodeLabel.newInstance("x", false),
- NodeLabel.newInstance("y", false)));
+ public Map<NodeId, Set<String>> getNodeToLabelsMap() {
+ Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
+ Set<String> setNodeLabels = new HashSet<String>(Arrays.asList("x", "y"));
map.put(NodeId.newInstance("host", 0), setNodeLabels);
return map;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java
index 227abe9..54d454d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeLabelsResponsePBImpl.java
@@ -72,11 +72,15 @@ public class GetClusterNodeLabelsResponsePBImpl extends
private void addNodeLabelsToProto() {
maybeInitBuilder();
builder.clearNodeLabels();
+ builder.clearDeprecatedNodeLabels();
List<NodeLabelProto> protoList = new ArrayList<NodeLabelProto>();
+ List<String> protoListString = new ArrayList<String>();
for (NodeLabel r : this.updatedNodeLabels) {
protoList.add(convertToProtoFormat(r));
+ protoListString.add(r.getName());
}
builder.addAllNodeLabels(protoList);
+ builder.addAllDeprecatedNodeLabels(protoListString);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
index 418fcbd..961b9f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
@@ -46,7 +46,7 @@ public class GetLabelsToNodesResponsePBImpl extends
GetLabelsToNodesResponseProto.Builder builder = null;
boolean viaProto = false;
- private Map<NodeLabel, Set<NodeId>> labelsToNodes;
+ private Map<String, Set<NodeId>> labelsToNodes;
public GetLabelsToNodesResponsePBImpl() {
this.builder = GetLabelsToNodesResponseProto.newBuilder();
@@ -63,7 +63,7 @@ public class GetLabelsToNodesResponsePBImpl extends
}
GetLabelsToNodesResponseProtoOrBuilder p = viaProto ? proto : builder;
List<LabelsToNodeIdsProto> list = p.getLabelsToNodesList();
- this.labelsToNodes = new HashMap<NodeLabel, Set<NodeId>>();
+ this.labelsToNodes = new HashMap<String, Set<NodeId>>();
for (LabelsToNodeIdsProto c : list) {
Set<NodeId> setNodes = new HashSet<NodeId>();
@@ -73,7 +73,7 @@ public class GetLabelsToNodesResponsePBImpl extends
}
if (!setNodes.isEmpty()) {
this.labelsToNodes
- .put(new NodeLabelPBImpl(c.getNodeLabels()), setNodes);
+ .put(c.getNodeLabels(), setNodes);
}
}
}
@@ -97,7 +97,7 @@ public class GetLabelsToNodesResponsePBImpl extends
public Iterator<LabelsToNodeIdsProto> iterator() {
return new Iterator<LabelsToNodeIdsProto>() {
- Iterator<Entry<NodeLabel, Set<NodeId>>> iter =
+ Iterator<Entry<String, Set<NodeId>>> iter =
labelsToNodes.entrySet().iterator();
@Override
@@ -107,13 +107,13 @@ public class GetLabelsToNodesResponsePBImpl extends
@Override
public LabelsToNodeIdsProto next() {
- Entry<NodeLabel, Set<NodeId>> now = iter.next();
+ Entry<String, Set<NodeId>> now = iter.next();
Set<NodeIdProto> nodeProtoSet = new HashSet<NodeIdProto>();
for(NodeId n : now.getValue()) {
nodeProtoSet.add(convertToProtoFormat(n));
}
return LabelsToNodeIdsProto.newBuilder()
- .setNodeLabels(convertToProtoFormat(now.getKey()))
+ .setNodeLabels(now.getKey())
.addAllNodeId(nodeProtoSet)
.build();
}
@@ -153,10 +153,6 @@ public class GetLabelsToNodesResponsePBImpl extends
return ((NodeIdPBImpl)t).getProto();
}
- private NodeLabelProto convertToProtoFormat(NodeLabel l) {
- return ((NodeLabelPBImpl)l).getProto();
- }
-
@Override
public int hashCode() {
assert false : "hashCode not designed";
@@ -176,7 +172,7 @@ public class GetLabelsToNodesResponsePBImpl extends
@Override
@Public
@Evolving
- public void setLabelsToNodes(Map<NodeLabel, Set<NodeId>> map) {
+ public void setLabelsToNodes(Map<String, Set<NodeId>> map) {
initLabelsToNodes();
labelsToNodes.clear();
labelsToNodes.putAll(map);
@@ -185,7 +181,7 @@ public class GetLabelsToNodesResponsePBImpl extends
@Override
@Public
@Evolving
- public Map<NodeLabel, Set<NodeId>> getLabelsToNodes() {
+ public Map<String, Set<NodeId>> getLabelsToNodes() {
initLabelsToNodes();
return this.labelsToNodes;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
index 52be73f..26a6389 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
@@ -32,11 +32,13 @@ import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsInfoProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProtoOrBuilder;
+import com.google.common.collect.Sets;
+
public class GetNodesToLabelsResponsePBImpl extends
GetNodesToLabelsResponse {
GetNodesToLabelsResponseProto proto = GetNodesToLabelsResponseProto
@@ -44,7 +46,7 @@ public class GetNodesToLabelsResponsePBImpl extends
GetNodesToLabelsResponseProto.Builder builder = null;
boolean viaProto = false;
- private Map<NodeId, Set<NodeLabel>> nodeToLabels;
+ private Map<NodeId, Set<String>> nodeToLabels;
public GetNodesToLabelsResponsePBImpl() {
this.builder = GetNodesToLabelsResponseProto.newBuilder();
@@ -60,15 +62,12 @@ public class GetNodesToLabelsResponsePBImpl extends
return;
}
GetNodesToLabelsResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<NodeIdToLabelsInfoProto> list = p.getNodeToLabelsList();
- this.nodeToLabels = new HashMap<NodeId, Set<NodeLabel>>();
-
- for (NodeIdToLabelsInfoProto c : list) {
- Set<NodeLabel> labels = new HashSet<NodeLabel>();
- for (NodeLabelProto l : c.getNodeLabelsList()) {
- labels.add(new NodeLabelPBImpl(l));
- }
- this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), labels);
+ List<NodeIdToLabelsProto> list = p.getNodeToLabelsList();
+ this.nodeToLabels = new HashMap<NodeId, Set<String>>();
+
+ for (NodeIdToLabelsProto c : list) {
+ this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()),
+ Sets.newHashSet(c.getNodeLabelsList()));
}
}
@@ -85,13 +84,13 @@ public class GetNodesToLabelsResponsePBImpl extends
if (nodeToLabels == null) {
return;
}
- Iterable<NodeIdToLabelsInfoProto> iterable =
- new Iterable<NodeIdToLabelsInfoProto>() {
+ Iterable<NodeIdToLabelsProto> iterable =
+ new Iterable<NodeIdToLabelsProto>() {
@Override
- public Iterator<NodeIdToLabelsInfoProto> iterator() {
- return new Iterator<NodeIdToLabelsInfoProto>() {
+ public Iterator<NodeIdToLabelsProto> iterator() {
+ return new Iterator<NodeIdToLabelsProto>() {
- Iterator<Entry<NodeId, Set<NodeLabel>>> iter = nodeToLabels
+ Iterator<Entry<NodeId, Set<String>>> iter = nodeToLabels
.entrySet().iterator();
@Override
@@ -100,16 +99,11 @@ public class GetNodesToLabelsResponsePBImpl extends
}
@Override
- public NodeIdToLabelsInfoProto next() {
- Entry<NodeId, Set<NodeLabel>> now = iter.next();
- Set<NodeLabelProto> labelProtoList =
- new HashSet<NodeLabelProto>();
- for (NodeLabel l : now.getValue()) {
- labelProtoList.add(convertToProtoFormat(l));
- }
- return NodeIdToLabelsInfoProto.newBuilder()
+ public NodeIdToLabelsProto next() {
+ Entry<NodeId, Set<String>> now = iter.next();
+ return NodeIdToLabelsProto.newBuilder()
.setNodeId(convertToProtoFormat(now.getKey()))
- .addAllNodeLabels(labelProtoList).build();
+ .addAllNodeLabels(now.getValue()).build();
}
@Override
@@ -144,13 +138,13 @@ public class GetNodesToLabelsResponsePBImpl extends
}
@Override
- public Map<NodeId, Set<NodeLabel>> getNodeToLabels() {
+ public Map<NodeId, Set<String>> getNodeToLabels() {
initNodeToLabels();
return this.nodeToLabels;
}
@Override
- public void setNodeToLabels(Map<NodeId, Set<NodeLabel>> map) {
+ public void setNodeToLabels(Map<NodeId, Set<String>> map) {
initNodeToLabels();
nodeToLabels.clear();
nodeToLabels.putAll(map);
@@ -159,10 +153,6 @@ public class GetNodesToLabelsResponsePBImpl extends
private NodeIdProto convertToProtoFormat(NodeId t) {
return ((NodeIdPBImpl)t).getProto();
}
-
- private NodeLabelProto convertToProtoFormat(NodeLabel t) {
- return ((NodeLabelPBImpl)t).getProto();
- }
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java
index 1ff0bef..0f1e400 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
import org.apache.hadoop.yarn.api.records.NodeLabel;
@@ -30,7 +29,7 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddTo
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
public class AddToClusterNodeLabelsRequestPBImpl extends
- AddToClusterNodeLabelsRequest {
+ AddToClusterNodeLabelsRequest {
AddToClusterNodeLabelsRequestProto proto = AddToClusterNodeLabelsRequestProto
.getDefaultInstance();
AddToClusterNodeLabelsRequestProto.Builder builder = null;
@@ -71,11 +70,15 @@ public class AddToClusterNodeLabelsRequestPBImpl extends
private void addNodeLabelsToProto() {
maybeInitBuilder();
builder.clearNodeLabels();
+ builder.clearDeprecatedNodeLabels();
List<NodeLabelProto> protoList = new ArrayList<NodeLabelProto>();
+ List<String> protoListString = new ArrayList<String>();
for (NodeLabel r : this.updatedNodeLabels) {
protoList.add(convertToProtoFormat(r));
+ protoListString.add(r.getName());
}
builder.addAllNodeLabels(protoList);
+ builder.addAllDeprecatedNodeLabels(protoListString);
}
@Override
@@ -101,17 +104,6 @@ public class AddToClusterNodeLabelsRequestPBImpl extends
viaProto = false;
}
- @Override
- public void setNodeLabels(List<NodeLabel> updatedNodeLabels) {
- maybeInitBuilder();
- this.updatedNodeLabels = new ArrayList<>();
- if (updatedNodeLabels == null) {
- builder.clearNodeLabels();
- return;
- }
- this.updatedNodeLabels.addAll(updatedNodeLabels);
- }
-
private void initLocalNodeLabels() {
AddToClusterNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder;
List<NodeLabelProto> attributesProtoList = p.getNodeLabelsList();
@@ -121,15 +113,6 @@ public class AddToClusterNodeLabelsRequestPBImpl extends
}
}
- @Override
- public List<NodeLabel> getNodeLabels() {
- if (this.updatedNodeLabels != null) {
- return this.updatedNodeLabels;
- }
- initLocalNodeLabels();
- return this.updatedNodeLabels;
- }
-
private NodeLabel convertFromProtoFormat(NodeLabelProto p) {
return new NodeLabelPBImpl(p);
}
@@ -142,4 +125,24 @@ public class AddToClusterNodeLabelsRequestPBImpl extends
public String toString() {
return getProto().toString();
}
+
+ @Override
+ public void setNodeLabels(List<NodeLabel> nodeLabels) {
+ maybeInitBuilder();
+ this.updatedNodeLabels = new ArrayList<>();
+ if (nodeLabels == null) {
+ builder.clearNodeLabels();
+ return;
+ }
+ this.updatedNodeLabels.addAll(nodeLabels);
+ }
+
+ @Override
+ public List<NodeLabel> getNodeLabels() {
+ if (this.updatedNodeLabels != null) {
+ return this.updatedNodeLabels;
+ }
+ initLocalNodeLabels();
+ return this.updatedNodeLabels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
index 3b15b27..f6634b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
@@ -28,7 +28,7 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsNameProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
@@ -58,10 +58,10 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends
return;
}
ReplaceLabelsOnNodeRequestProtoOrBuilder p = viaProto ? proto : builder;
- List<NodeIdToLabelsNameProto> list = p.getNodeToLabelsList();
+ List<NodeIdToLabelsProto> list = p.getNodeToLabelsList();
this.nodeIdToLabels = new HashMap<NodeId, Set<String>>();
- for (NodeIdToLabelsNameProto c : list) {
+ for (NodeIdToLabelsProto c : list) {
this.nodeIdToLabels.put(new NodeIdPBImpl(c.getNodeId()),
Sets.newHashSet(c.getNodeLabelsList()));
}
@@ -80,11 +80,11 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends
if (nodeIdToLabels == null) {
return;
}
- Iterable<NodeIdToLabelsNameProto> iterable =
- new Iterable<NodeIdToLabelsNameProto>() {
+ Iterable<NodeIdToLabelsProto> iterable =
+ new Iterable<NodeIdToLabelsProto>() {
@Override
- public Iterator<NodeIdToLabelsNameProto> iterator() {
- return new Iterator<NodeIdToLabelsNameProto>() {
+ public Iterator<NodeIdToLabelsProto> iterator() {
+ return new Iterator<NodeIdToLabelsProto>() {
Iterator<Entry<NodeId, Set<String>>> iter = nodeIdToLabels
.entrySet().iterator();
@@ -95,9 +95,9 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends
}
@Override
- public NodeIdToLabelsNameProto next() {
+ public NodeIdToLabelsProto next() {
Entry<NodeId, Set<String>> now = iter.next();
- return NodeIdToLabelsNameProto.newBuilder()
+ return NodeIdToLabelsProto.newBuilder()
.setNodeId(convertToProtoFormat(now.getKey())).clearNodeLabels()
.addAllNodeLabels(now.getValue()).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 86bb896..9dda57e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -735,9 +735,10 @@ public class AdminService extends CompositeService implements
AddToClusterNodeLabelsResponse response =
recordFactory.newRecordInstance(AddToClusterNodeLabelsResponse.class);
try {
- rmContext.getNodeLabelManager().addToCluserNodeLabels(request.getNodeLabels());
- RMAuditLogger
- .logSuccess(user.getShortUserName(), operation, "AdminService");
+ rmContext.getNodeLabelManager()
+ .addToCluserNodeLabels(request.getNodeLabels());
+ RMAuditLogger.logSuccess(user.getShortUserName(), operation,
+ "AdminService");
return response;
} catch (IOException ioe) {
throw logAndWrapException(ioe, user.getShortUserName(), operation, msg);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index c4e3bf7..0c87ede 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1490,7 +1490,7 @@ public class ClientRMService extends AbstractService implements
GetNodesToLabelsRequest request) throws YarnException, IOException {
RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
GetNodesToLabelsResponse response =
- GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabelsInfo());
+ GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels());
return response;
}
@@ -1500,10 +1500,10 @@ public class ClientRMService extends AbstractService implements
RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
if (request.getNodeLabels() == null || request.getNodeLabels().isEmpty()) {
return GetLabelsToNodesResponse.newInstance(
- labelsMgr.getLabelsInfoToNodes());
+ labelsMgr.getLabelsToNodes());
} else {
return GetLabelsToNodesResponse.newInstance(
- labelsMgr.getLabelsInfoToNodes(request.getNodeLabels()));
+ labelsMgr.getLabelsToNodes(request.getNodeLabels()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a6ca75f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 6376290..d5f9acc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1799,27 +1799,19 @@ public class TestClientRMService {
// Get node labels mapping
GetNodesToLabelsResponse response1 = client
.getNodeToLabels(GetNodesToLabelsRequest.newInstance());
- Map<NodeId, Set<NodeLabel>> nodeToLabels = response1.getNodeToLabels();
+ Map<NodeId, Set<String>> nodeToLabels = response1.getNodeToLabels();
Assert.assertTrue(nodeToLabels.keySet().containsAll(
Arrays.asList(node1, node2)));
Assert.assertTrue(nodeToLabels.get(node1)
- .containsAll(Arrays.asList(labelX)));
+ .containsAll(Arrays.asList(labelX.getName())));
Assert.assertTrue(nodeToLabels.get(node2)
- .containsAll(Arrays.asList(labelY)));
- // Verify whether labelX's exclusivity is false
- for (NodeLabel x : nodeToLabels.get(node1)) {
- Assert.assertFalse(x.isExclusive());
- }
- // Verify whether labelY's exclusivity is true
- for (NodeLabel y : nodeToLabels.get(node2)) {
- Assert.assertTrue(y.isExclusive());
- }
+ .containsAll(Arrays.asList(labelY.getName())));
// Below label "x" is not present in the response as exclusivity is true
Assert.assertFalse(nodeToLabels.get(node1).containsAll(
Arrays.asList(NodeLabel.newInstance("x"))));
rpc.stopProxy(client, conf);
- rm.close();
+ rm.stop();
}
@Test
@@ -1870,18 +1862,14 @@ public class TestClientRMService {
// Get labels to nodes mapping
GetLabelsToNodesResponse response1 = client
.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
- Map<NodeLabel, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
- // Verify whether all NodeLabel's exclusivity are false
- for (Map.Entry<NodeLabel, Set<NodeId>> nltn : labelsToNodes.entrySet()) {
- Assert.assertFalse(nltn.getKey().isExclusive());
- }
+ Map<String, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
Assert.assertTrue(labelsToNodes.keySet().containsAll(
- Arrays.asList(labelX, labelY, labelZ)));
- Assert.assertTrue(labelsToNodes.get(labelX).containsAll(
+ Arrays.asList(labelX.getName(), labelY.getName(), labelZ.getName())));
+ Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(
Arrays.asList(node1A)));
- Assert.assertTrue(labelsToNodes.get(labelY).containsAll(
+ Assert.assertTrue(labelsToNodes.get(labelY.getName()).containsAll(
Arrays.asList(node2A, node3A)));
- Assert.assertTrue(labelsToNodes.get(labelZ).containsAll(
+ Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(
Arrays.asList(node1B, node3B)));
// Get labels to nodes mapping for specific labels
@@ -1890,17 +1878,13 @@ public class TestClientRMService {
GetLabelsToNodesResponse response2 = client
.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
labelsToNodes = response2.getLabelsToNodes();
- // Verify whether all NodeLabel's exclusivity are false
- for (Map.Entry<NodeLabel, Set<NodeId>> nltn : labelsToNodes.entrySet()) {
- Assert.assertFalse(nltn.getKey().isExclusive());
- }
Assert.assertTrue(labelsToNodes.keySet().containsAll(
- Arrays.asList(labelX, labelZ)));
- Assert.assertTrue(labelsToNodes.get(labelX).containsAll(
+ Arrays.asList(labelX.getName(), labelZ.getName())));
+ Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(
Arrays.asList(node1A)));
- Assert.assertTrue(labelsToNodes.get(labelZ).containsAll(
+ Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(
Arrays.asList(node1B, node3B)));
- Assert.assertEquals(labelsToNodes.get(labelY), null);
+ Assert.assertEquals(labelsToNodes.get(labelY.getName()), null);
rpc.stopProxy(client, conf);
rm.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-6194. Cluster capacity in
SchedulingPolicy is updated only on allocation file reload. (Yufei Gu via
kasha)
Posted by su...@apache.org.
YARN-6194. Cluster capacity in SchedulingPolicy is updated only on allocation file reload. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b10e9622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b10e9622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b10e9622
Branch: refs/heads/YARN-2915
Commit: b10e962224a8ae1c6031a05322b0cc5e564bd078
Parents: 718ad9f
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Wed Feb 22 15:58:49 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Wed Feb 22 15:58:49 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSContext.java | 21 ++++++++++++----
.../resourcemanager/scheduler/fair/FSQueue.java | 2 +-
.../scheduler/fair/FairScheduler.java | 6 ++---
.../scheduler/fair/SchedulingPolicy.java | 19 ++++++++++++++-
.../DominantResourceFairnessPolicy.java | 16 +++++++------
.../scheduler/fair/TestFairScheduler.java | 8 +++----
.../TestDominantResourceFairnessPolicy.java | 25 +++++++++++++++++++-
7 files changed, 74 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
index 56bc99c..a4aa8f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSContext.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+import org.apache.hadoop.yarn.api.records.Resource;
+
/**
* Helper class that holds basic information to be passed around
* FairScheduler classes. Think of this as a glorified map that holds key
@@ -27,28 +29,37 @@ public class FSContext {
private boolean preemptionEnabled = false;
private float preemptionUtilizationThreshold;
private FSStarvedApps starvedApps;
+ private FairScheduler scheduler;
+
+ FSContext(FairScheduler scheduler) {
+ this.scheduler = scheduler;
+ }
- public boolean isPreemptionEnabled() {
+ boolean isPreemptionEnabled() {
return preemptionEnabled;
}
- public void setPreemptionEnabled() {
+ void setPreemptionEnabled() {
this.preemptionEnabled = true;
if (starvedApps == null) {
starvedApps = new FSStarvedApps();
}
}
- public FSStarvedApps getStarvedApps() {
+ FSStarvedApps getStarvedApps() {
return starvedApps;
}
- public float getPreemptionUtilizationThreshold() {
+ float getPreemptionUtilizationThreshold() {
return preemptionUtilizationThreshold;
}
- public void setPreemptionUtilizationThreshold(
+ void setPreemptionUtilizationThreshold(
float preemptionUtilizationThreshold) {
this.preemptionUtilizationThreshold = preemptionUtilizationThreshold;
}
+
+ public Resource getClusterResource() {
+ return scheduler.getClusterResource();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 7e8b858..b5592c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -135,7 +135,7 @@ public abstract class FSQueue implements Queue, Schedulable {
}
public void setPolicy(SchedulingPolicy policy) {
- policy.initialize(scheduler.getClusterResource());
+ policy.initialize(scheduler.getContext());
this.policy = policy;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a15e6b5..c946bfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -205,13 +205,12 @@ public class FairScheduler extends
public FairScheduler() {
super(FairScheduler.class.getName());
- context = new FSContext();
+ context = new FSContext(this);
allocsLoader = new AllocationFileLoaderService();
queueMgr = new QueueManager(this);
maxRunningEnforcer = new MaxRunningAppsEnforcer(this);
}
- @VisibleForTesting
public FSContext getContext() {
return context;
}
@@ -1452,8 +1451,7 @@ public class FairScheduler extends
} else {
allocConf = queueInfo;
setQueueAcls(allocConf.getQueueAcls());
- allocConf.getDefaultSchedulingPolicy().initialize(
- getClusterResource());
+ allocConf.getDefaultSchedulingPolicy().initialize(getContext());
queueMgr.updateAllocationConfiguration(allocConf);
applyChildDefaults();
maxRunningEnforcer.updateRunnabilityOnReload();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
index 3fe36f3..9a9be8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
@@ -91,10 +91,27 @@ public abstract class SchedulingPolicy {
}
return getInstance(clazz);
}
-
+
+ /**
+ * Initialize the scheduling policy with cluster resources.
+ * @deprecated Since it doesn't track cluster resource changes, replaced by
+ * {@link #initialize(FSContext)}.
+ *
+ * @param clusterCapacity cluster resources
+ */
+ @Deprecated
public void initialize(Resource clusterCapacity) {}
/**
+ * Initialize the scheduling policy with a {@link FSContext} object, which has
+ * a pointer to the cluster resources among other information.
+ *
+ * @param fsContext a {@link FSContext} object which has a pointer to the
+ * cluster resources
+ */
+ public void initialize(FSContext fsContext) {}
+
+ /**
* The {@link ResourceCalculator} returned by this method should be used
* for any calculations involving resources.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 369b8a1..193ed4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
@@ -104,17 +105,17 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
}
@Override
- public void initialize(Resource clusterCapacity) {
- COMPARATOR.setClusterCapacity(clusterCapacity);
+ public void initialize(FSContext fsContext) {
+ COMPARATOR.setFSContext(fsContext);
}
public static class DominantResourceFairnessComparator implements Comparator<Schedulable> {
private static final int NUM_RESOURCES = ResourceType.values().length;
-
- private Resource clusterCapacity;
- public void setClusterCapacity(Resource clusterCapacity) {
- this.clusterCapacity = clusterCapacity;
+ private FSContext fsContext;
+
+ public void setFSContext(FSContext fsContext) {
+ this.fsContext = fsContext;
}
@Override
@@ -125,7 +126,8 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
ResourceWeights sharesOfMinShare2 = new ResourceWeights();
ResourceType[] resourceOrder1 = new ResourceType[NUM_RESOURCES];
ResourceType[] resourceOrder2 = new ResourceType[NUM_RESOURCES];
-
+ Resource clusterCapacity = fsContext.getClusterResource();
+
// Calculate shares of the cluster for each resource both schedulables.
calculateShares(s1.getResourceUsage(),
clusterCapacity, sharesOfCluster1, resourceOrder1, s1.getWeights());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 4def53f..62430bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3293,7 +3293,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
- drfPolicy.initialize(scheduler.getClusterResource());
+ drfPolicy.initialize(scheduler.getContext());
scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
scheduler.update();
@@ -3339,7 +3339,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
- drfPolicy.initialize(scheduler.getClusterResource());
+ drfPolicy.initialize(scheduler.getContext());
scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
scheduler.update();
@@ -3354,7 +3354,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
Assert.assertEquals(1, app2.getLiveContainers().size());
}
-
+
@Test
public void testDRFHierarchicalQueues() throws Exception {
scheduler.init(conf);
@@ -3384,7 +3384,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSAppAttempt app4 = scheduler.getSchedulerApp(appAttId4);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
- drfPolicy.initialize(scheduler.getClusterResource());
+ drfPolicy.initialize(scheduler.getContext());
scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
scheduler.getQueueManager().getQueue("queue1.subqueue1").setPolicy(drfPolicy);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b10e9622/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
index a5c20c1..3719e2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
@@ -19,12 +19,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.util.Comparator;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -40,7 +43,10 @@ public class TestDominantResourceFairnessPolicy {
private Comparator<Schedulable> createComparator(int clusterMem,
int clusterCpu) {
DominantResourceFairnessPolicy policy = new DominantResourceFairnessPolicy();
- policy.initialize(BuilderUtils.newResource(clusterMem, clusterCpu));
+ FSContext fsContext = mock(FSContext.class);
+ when(fsContext.getClusterResource()).
+ thenReturn(Resources.createResource(clusterMem, clusterCpu));
+ policy.initialize(fsContext);
return policy.getComparator();
}
@@ -160,4 +166,21 @@ public class TestDominantResourceFairnessPolicy {
assertEquals(ResourceType.CPU, resourceOrder[0]);
assertEquals(ResourceType.MEMORY, resourceOrder[1]);
}
+
+ @Test
+ public void testCompareSchedulablesWithClusterResourceChanges(){
+ Schedulable schedulable1 = createSchedulable(2000, 1);
+ Schedulable schedulable2 = createSchedulable(1000, 2);
+
+ // schedulable1 has share weights [1/2, 1/5], schedulable2 has share
+ // weights [1/4, 2/5], schedulable1 > schedulable2 since 1/2 > 2/5
+ assertTrue(createComparator(4000, 5)
+ .compare(schedulable1, schedulable2) > 0);
+
+ // share weights have changed because of the cluster resource change.
+ // schedulable1 has share weights [1/4, 1/6], schedulable2 has share
+ // weights [1/8, 1/3], schedulable1 < schedulable2 since 1/4 < 1/3
+ assertTrue(createComparator(8000, 6)
+ .compare(schedulable1, schedulable2) < 0);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-5407. In-memory based
implementation of the
FederationApplicationStateStore/FederationPolicyStateStore. (Ellen Hui via
Subru)
Posted by su...@apache.org.
YARN-5407. In-memory based implementation of the FederationApplicationStateStore/FederationPolicyStateStore. (Ellen Hui via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9abcba9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9abcba9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9abcba9
Branch: refs/heads/YARN-2915
Commit: d9abcba90edef213ed55fbfeddae820c4bee1d1b
Parents: ca9c223
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Aug 9 16:07:55 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../store/impl/MemoryFederationStateStore.java | 158 +++++++-
...SubClusterPoliciesConfigurationsRequest.java | 2 +-
...ubClusterPoliciesConfigurationsResponse.java | 2 +-
...GetSubClusterPolicyConfigurationRequest.java | 3 +-
...etSubClusterPolicyConfigurationResponse.java | 2 +-
...SetSubClusterPolicyConfigurationRequest.java | 20 +-
...etSubClusterPolicyConfigurationResponse.java | 2 +-
.../records/SubClusterPolicyConfiguration.java | 27 +-
...tApplicationHomeSubClusterRequestPBImpl.java | 4 +
...ClusterPolicyConfigurationRequestPBImpl.java | 17 -
.../pb/SubClusterPolicyConfigurationPBImpl.java | 17 +
.../proto/yarn_server_federation_protos.proto | 8 +-
.../impl/FederationStateStoreBaseTest.java | 367 ++++++++++++++++++-
.../impl/TestMemoryFederationStateStore.java | 4 +-
14 files changed, 558 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index cea4ac2..a540dff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -20,35 +20,72 @@ package org.apache.hadoop.yarn.server.federation.store.impl;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
/**
- * In-memory implementation of FederationMembershipStateStore.
+ * In-memory implementation of {@link FederationStateStore}.
*/
-public class MemoryFederationStateStore
- implements FederationMembershipStateStore {
+public class MemoryFederationStateStore implements FederationStateStore {
+
+ private Map<SubClusterId, SubClusterInfo> membership;
+ private Map<ApplicationId, SubClusterId> applications;
+ private Map<String, SubClusterPolicyConfiguration> policies;
- private final Map<SubClusterId, SubClusterInfo> membership =
- new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
private final MonotonicClock clock = new MonotonicClock();
@Override
+ public void init(Configuration conf) {
+ membership = new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
+ applications = new ConcurrentHashMap<ApplicationId, SubClusterId>();
+ policies = new ConcurrentHashMap<String, SubClusterPolicyConfiguration>();
+ }
+
+ @Override
+ public void close() {
+ membership = null;
+ applications = null;
+ policies = null;
+ }
+
+ @Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
@@ -116,4 +153,113 @@ public class MemoryFederationStateStore
return GetSubClustersInfoResponse.newInstance(result);
}
+ // FederationApplicationHomeSubClusterStore methods
+
+ @Override
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId =
+ request.getApplicationHomeSubCluster().getApplicationId();
+ if (applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " already exists");
+ }
+
+ applications.put(appId,
+ request.getApplicationHomeSubCluster().getHomeSubCluster());
+ return AddApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId =
+ request.getApplicationHomeSubCluster().getApplicationId();
+ if (!applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " does not exist");
+ }
+
+ applications.put(appId,
+ request.getApplicationHomeSubCluster().getHomeSubCluster());
+ return UpdateApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId = request.getApplicationId();
+ if (!applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " does not exist");
+ }
+
+ return GetApplicationHomeSubClusterResponse.newInstance(
+ ApplicationHomeSubCluster.newInstance(appId, applications.get(appId)));
+ }
+
+ @Override
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException {
+ List<ApplicationHomeSubCluster> result =
+ new ArrayList<ApplicationHomeSubCluster>();
+ for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
+ result
+ .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+ }
+
+ GetApplicationsHomeSubClusterResponse.newInstance(result);
+ return GetApplicationsHomeSubClusterResponse.newInstance(result);
+ }
+
+ @Override
+ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ DeleteApplicationHomeSubClusterRequest request) throws YarnException {
+ ApplicationId appId = request.getApplicationId();
+ if (!applications.containsKey(appId)) {
+ throw new YarnException("Application " + appId + " does not exist");
+ }
+
+ applications.remove(appId);
+ return DeleteApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ String queue = request.getQueue();
+ if (!policies.containsKey(queue)) {
+ throw new YarnException("Policy for queue " + queue + " does not exist");
+ }
+
+ return GetSubClusterPolicyConfigurationResponse
+ .newInstance(policies.get(queue));
+ }
+
+ @Override
+ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ policies.put(request.getPolicyConfiguration().getQueue(),
+ request.getPolicyConfiguration());
+ return SetSubClusterPolicyConfigurationResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException {
+ ArrayList<SubClusterPolicyConfiguration> result =
+ new ArrayList<SubClusterPolicyConfiguration>();
+ for (SubClusterPolicyConfiguration policy : policies.values()) {
+ result.add(policy);
+ }
+ return GetSubClusterPoliciesConfigurationsResponse.newInstance(result);
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return null;
+ }
+
+ @Override
+ public Version loadVersion() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
index 404521b..8cb84f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.util.Records;
@Private
@Unstable
public abstract class GetSubClusterPoliciesConfigurationsRequest {
- public GetSubClusterPoliciesConfigurationsRequest newInstance() {
+ public static GetSubClusterPoliciesConfigurationsRequest newInstance() {
return Records.newRecord(GetSubClusterPoliciesConfigurationsRequest.class);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
index 6554d68..2eaeb51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
@@ -36,7 +36,7 @@ public abstract class GetSubClusterPoliciesConfigurationsResponse {
@Private
@Unstable
- public GetSubClusterPoliciesConfigurationsResponse newInstance(
+ public static GetSubClusterPoliciesConfigurationsResponse newInstance(
List<SubClusterPolicyConfiguration> policyConfigurations) {
GetSubClusterPoliciesConfigurationsResponse response =
Records.newRecord(GetSubClusterPoliciesConfigurationsResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
index 7b7d8c4..c3f49e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
@@ -33,7 +33,8 @@ public abstract class GetSubClusterPolicyConfigurationRequest {
@Private
@Unstable
- public GetSubClusterPolicyConfigurationRequest newInstance(String queueName) {
+ public static GetSubClusterPolicyConfigurationRequest newInstance(
+ String queueName) {
GetSubClusterPolicyConfigurationRequest request =
Records.newRecord(GetSubClusterPolicyConfigurationRequest.class);
request.setQueue(queueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
index 11a46e0..350b239 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
@@ -34,7 +34,7 @@ public abstract class GetSubClusterPolicyConfigurationResponse {
@Private
@Unstable
- public GetSubClusterPolicyConfigurationResponse newInstance(
+ public static GetSubClusterPolicyConfigurationResponse newInstance(
SubClusterPolicyConfiguration policy) {
GetSubClusterPolicyConfigurationResponse response =
Records.newRecord(GetSubClusterPolicyConfigurationResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
index 06d5399..743ad0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class SetSubClusterPolicyConfigurationRequest {
@Private
@Unstable
- public SetSubClusterPolicyConfigurationRequest newInstance(
+ public static SetSubClusterPolicyConfigurationRequest newInstance(
SubClusterPolicyConfiguration policy) {
SetSubClusterPolicyConfigurationRequest request =
Records.newRecord(SetSubClusterPolicyConfigurationRequest.class);
@@ -41,24 +41,6 @@ public abstract class SetSubClusterPolicyConfigurationRequest {
}
/**
- * Get the name of the queue for which we are configuring a policy.
- *
- * @return the name of the queue
- */
- @Public
- @Unstable
- public abstract String getQueue();
-
- /**
- * Sets the name of the queue for which we are configuring a policy.
- *
- * @param queueName the name of the queue
- */
- @Private
- @Unstable
- public abstract void setQueue(String queueName);
-
- /**
* Get the policy configuration assigned to the queue.
*
* @return the policy for the specified queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
index 33c4043..401e984 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.util.Records;
@Private
@Unstable
public abstract class SetSubClusterPolicyConfigurationResponse {
- public SetSubClusterPolicyConfigurationResponse newInstance() {
+ public static SetSubClusterPolicyConfigurationResponse newInstance() {
return Records.newRecord(SetSubClusterPolicyConfigurationResponse.class);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
index bc12acb..2839139 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -29,8 +29,8 @@ import java.nio.ByteBuffer;
/**
* {@link SubClusterPolicyConfiguration} is a class that represents a
- * configuration of a policy. It contains a policy type (resolve to a class
- * name) and its params as an opaque {@link ByteBuffer}.
+ * configuration of a policy. For a single queue, it contains a policy type
+ * (resolve to a class name) and its params as an opaque {@link ByteBuffer}.
*
* Note: by design the params are an opaque ByteBuffer, this allows for enough
* flexibility to evolve the policies without impacting the protocols to/from
@@ -42,16 +42,35 @@ public abstract class SubClusterPolicyConfiguration {
@Private
@Unstable
- public static SubClusterPolicyConfiguration newInstance(String policyType,
- ByteBuffer policyParams) {
+ public static SubClusterPolicyConfiguration newInstance(String queue,
+ String policyType, ByteBuffer policyParams) {
SubClusterPolicyConfiguration policy =
Records.newRecord(SubClusterPolicyConfiguration.class);
+ policy.setQueue(queue);
policy.setType(policyType);
policy.setParams(policyParams);
return policy;
}
/**
+ * Get the name of the queue for which we are configuring a policy.
+ *
+ * @return the name of the queue
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Sets the name of the queue for which we are configuring a policy.
+ *
+ * @param queueName the name of the queue
+ */
+ @Private
+ @Unstable
+ public abstract void setQueue(String queueName);
+
+ /**
* Get the type of the policy. This could be random, round-robin, load-based,
* etc.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
index 865d0c4..585ba81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
@@ -108,6 +108,10 @@ public class GetApplicationHomeSubClusterRequestPBImpl
public ApplicationId getApplicationId() {
GetApplicationHomeSubClusterRequestProtoOrBuilder p =
viaProto ? proto : builder;
+ if (applicationId != null) {
+ return applicationId;
+ }
+
if (!p.hasApplicationId()) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
index 5e29bd5..7b7f89d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -107,23 +107,6 @@ public class SetSubClusterPolicyConfigurationRequestPBImpl
}
@Override
- public String getQueue() {
- SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
- viaProto ? proto : builder;
- return p.getQueue();
- }
-
- @Override
- public void setQueue(String queueName) {
- maybeInitBuilder();
- if (queueName == null) {
- builder.clearQueue();
- return;
- }
- builder.setQueue(queueName);
- }
-
- @Override
public SubClusterPolicyConfiguration getPolicyConfiguration() {
SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
viaProto ? proto : builder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
index fe9d9db..305a8d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
@@ -87,6 +87,23 @@ public class SubClusterPolicyConfigurationPBImpl
}
@Override
+ public String getQueue() {
+ SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearType();
+ return;
+ }
+ builder.setQueue(queueName);
+
+ }
+
+ @Override
public String getType() {
SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
return p.getType();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 3f1cee9..11f786f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -136,8 +136,9 @@ message DeleteApplicationHomeSubClusterResponseProto {
}
message SubClusterPolicyConfigurationProto {
- optional string type = 1;
- optional bytes params = 2;
+ optional string queue = 1;
+ optional string type = 2;
+ optional bytes params = 3;
}
message GetSubClusterPolicyConfigurationRequestProto {
@@ -149,8 +150,7 @@ message GetSubClusterPolicyConfigurationResponseProto {
}
message SetSubClusterPolicyConfigurationRequestProto {
- optional string queue = 1;
- optional SubClusterPolicyConfigurationProto policy_configuration = 2;
+ optional SubClusterPolicyConfigurationProto policy_configuration = 1;
}
message SetSubClusterPolicyConfigurationResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index c76a485..165dd78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -18,18 +18,39 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
import org.apache.hadoop.yarn.util.MonotonicClock;
import org.junit.After;
import org.junit.Assert;
@@ -42,20 +63,21 @@ import org.junit.Test;
public abstract class FederationStateStoreBaseTest {
private static final MonotonicClock CLOCK = new MonotonicClock();
+ private FederationStateStore stateStore = createStateStore();
- private FederationMembershipStateStore stateStore;
+ protected abstract FederationStateStore createStateStore();
@Before
- public void before() throws IOException {
- stateStore = getCleanStateStore();
+ public void before() throws IOException, YarnException {
+ stateStore.init(new Configuration());
}
@After
- public void after() {
- stateStore = null;
+ public void after() throws Exception {
+ stateStore.close();
}
- protected abstract FederationMembershipStateStore getCleanStateStore();
+ // Test FederationMembershipStateStore
@Test
public void testRegisterSubCluster() throws Exception {
@@ -72,10 +94,7 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testDeregisterSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
-
- stateStore.registerSubCluster(
- SubClusterRegisterRequest.newInstance(subClusterInfo));
+ registerSubCluster(subClusterId);
SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
.newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
@@ -105,9 +124,7 @@ public abstract class FederationStateStoreBaseTest {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
-
- stateStore.registerSubCluster(
- SubClusterRegisterRequest.newInstance(subClusterInfo));
+ registerSubCluster(subClusterId);
GetSubClusterInfoRequest request =
GetSubClusterInfoRequest.newInstance(subClusterId);
@@ -167,10 +184,7 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testSubClusterHeartbeat() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
-
- stateStore.registerSubCluster(
- SubClusterRegisterRequest.newInstance(subClusterInfo));
+ registerSubCluster(subClusterId);
SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
.newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
@@ -196,6 +210,271 @@ public abstract class FederationStateStoreBaseTest {
}
}
+ // Test FederationApplicationHomeSubClusterStore
+
+ @Test
+ public void testAddApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc);
+ AddApplicationHomeSubClusterResponse response =
+ stateStore.addApplicationHomeSubClusterMap(request);
+
+ Assert.assertNotNull(response);
+ Assert.assertEquals(subClusterId, queryApplicationHomeSC(appId));
+
+ }
+
+ @Test
+ public void testAddApplicationHomeSubClusterMapAppAlreadyExists()
+ throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ addApplicationHomeSC(appId, subClusterId1);
+
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ ApplicationHomeSubCluster ahsc2 =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
+
+ try {
+ stateStore.addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc2));
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " already exists"));
+ }
+
+ Assert.assertEquals(subClusterId1, queryApplicationHomeSC(appId));
+
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ addApplicationHomeSC(appId, subClusterId);
+
+ DeleteApplicationHomeSubClusterRequest delRequest =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+
+ DeleteApplicationHomeSubClusterResponse response =
+ stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+
+ Assert.assertNotNull(response);
+ try {
+ queryApplicationHomeSC(appId);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId + " does not exist"));
+ }
+
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterMapUnknownApp()
+ throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ DeleteApplicationHomeSubClusterRequest delRequest =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+
+ try {
+ stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ addApplicationHomeSC(appId, subClusterId);
+
+ GetApplicationHomeSubClusterRequest getRequest =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ GetApplicationHomeSubClusterResponse result =
+ stateStore.getApplicationHomeSubClusterMap(getRequest);
+
+ Assert.assertEquals(appId,
+ result.getApplicationHomeSubCluster().getApplicationId());
+ Assert.assertEquals(subClusterId,
+ result.getApplicationHomeSubCluster().getHomeSubCluster());
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterMapUnknownApp() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ try {
+ stateStore.getApplicationHomeSubClusterMap(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetApplicationsHomeSubClusterMap() throws Exception {
+ ApplicationId appId1 = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ ApplicationHomeSubCluster ahsc1 =
+ ApplicationHomeSubCluster.newInstance(appId1, subClusterId1);
+
+ ApplicationId appId2 = ApplicationId.newInstance(1, 2);
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ ApplicationHomeSubCluster ahsc2 =
+ ApplicationHomeSubCluster.newInstance(appId2, subClusterId2);
+
+ addApplicationHomeSC(appId1, subClusterId1);
+ addApplicationHomeSC(appId2, subClusterId2);
+
+ GetApplicationsHomeSubClusterRequest getRequest =
+ GetApplicationsHomeSubClusterRequest.newInstance();
+
+ GetApplicationsHomeSubClusterResponse result =
+ stateStore.getApplicationsHomeSubClusterMap(getRequest);
+
+ Assert.assertEquals(2, result.getAppsHomeSubClusters().size());
+ Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc1));
+ Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc2));
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterMap() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ addApplicationHomeSC(appId, subClusterId1);
+
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ ApplicationHomeSubCluster ahscUpdate =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
+
+ UpdateApplicationHomeSubClusterRequest updateRequest =
+ UpdateApplicationHomeSubClusterRequest.newInstance(ahscUpdate);
+
+ UpdateApplicationHomeSubClusterResponse response =
+ stateStore.updateApplicationHomeSubClusterMap(updateRequest);
+
+ Assert.assertNotNull(response);
+
+ Assert.assertEquals(subClusterId2, queryApplicationHomeSC(appId));
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterMapUnknownApp()
+ throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId1);
+
+ UpdateApplicationHomeSubClusterRequest updateRequest =
+ UpdateApplicationHomeSubClusterRequest.newInstance(ahsc);
+
+ try {
+ stateStore.updateApplicationHomeSubClusterMap((updateRequest));
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Application " + appId.toString() + " does not exist"));
+ }
+ }
+
+ // Test FederationPolicyStore
+
+ @Test
+ public void testSetPolicyConfiguration() throws Exception {
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf("Queue", "PolicyType"));
+
+ SetSubClusterPolicyConfigurationResponse result =
+ stateStore.setPolicyConfiguration(request);
+
+ Assert.assertNotNull(result);
+ Assert.assertEquals(createSCPolicyConf("Queue", "PolicyType"),
+ queryPolicy("Queue"));
+
+ }
+
+ @Test
+ public void testSetPolicyConfigurationUpdateExisting() throws Exception {
+ setPolicyConf("Queue", "PolicyType1");
+
+ SetSubClusterPolicyConfigurationRequest request2 =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf("Queue", "PolicyType2"));
+ SetSubClusterPolicyConfigurationResponse result =
+ stateStore.setPolicyConfiguration(request2);
+
+ Assert.assertNotNull(result);
+ Assert.assertEquals(createSCPolicyConf("Queue", "PolicyType2"),
+ queryPolicy("Queue"));
+ }
+
+ @Test
+ public void testGetPolicyConfiguration() throws Exception {
+ setPolicyConf("Queue", "PolicyType");
+
+ GetSubClusterPolicyConfigurationRequest getRequest =
+ GetSubClusterPolicyConfigurationRequest.newInstance("Queue");
+ GetSubClusterPolicyConfigurationResponse result =
+ stateStore.getPolicyConfiguration(getRequest);
+
+ Assert.assertNotNull(result);
+ Assert.assertEquals(createSCPolicyConf("Queue", "PolicyType"),
+ result.getPolicyConfiguration());
+
+ }
+
+ @Test
+ public void testGetPolicyConfigurationUnknownQueue() throws Exception {
+
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance("Queue");
+ try {
+ stateStore.getPolicyConfiguration(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Policy for queue Queue does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetPoliciesConfigurations() throws Exception {
+ setPolicyConf("Queue1", "PolicyType1");
+ setPolicyConf("Queue2", "PolicyType2");
+
+ GetSubClusterPoliciesConfigurationsResponse response =
+ stateStore.getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest.newInstance());
+
+ Assert.assertNotNull(response);
+ Assert.assertNotNull(response.getPoliciesConfigs());
+
+ Assert.assertEquals(2, response.getPoliciesConfigs().size());
+
+ Assert.assertTrue(response.getPoliciesConfigs()
+ .contains(createSCPolicyConf("Queue1", "PolicyType1")));
+ Assert.assertTrue(response.getPoliciesConfigs()
+ .contains(createSCPolicyConf("Queue2", "PolicyType2")));
+ }
+
+ // Convenience methods
+
private SubClusterInfo createSubClusterInfo(SubClusterId subClusterId) {
String amRMAddress = "1.2.3.4:1";
@@ -208,6 +487,37 @@ public abstract class FederationStateStoreBaseTest {
CLOCK.getTime(), "cabability");
}
+ private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
+ String policyType) {
+ return SubClusterPolicyConfiguration.newInstance(queueName, policyType,
+ ByteBuffer.allocate(1));
+ }
+
+ private void addApplicationHomeSC(ApplicationId appId,
+ SubClusterId subClusterId) throws YarnException {
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc);
+ stateStore.addApplicationHomeSubClusterMap(request);
+ }
+
+ private void setPolicyConf(String queue, String policyType)
+ throws YarnException {
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf(queue, policyType));
+ stateStore.setPolicyConfiguration(request);
+ }
+
+ private void registerSubCluster(SubClusterId subClusterId)
+ throws YarnException {
+
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ }
+
private SubClusterInfo querySubClusterInfo(SubClusterId subClusterId)
throws YarnException {
GetSubClusterInfoRequest request =
@@ -215,4 +525,25 @@ public abstract class FederationStateStoreBaseTest {
return stateStore.getSubCluster(request).getSubClusterInfo();
}
+ private SubClusterId queryApplicationHomeSC(ApplicationId appId)
+ throws YarnException {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ GetApplicationHomeSubClusterResponse response =
+ stateStore.getApplicationHomeSubClusterMap(request);
+
+ return response.getApplicationHomeSubCluster().getHomeSubCluster();
+ }
+
+ private SubClusterPolicyConfiguration queryPolicy(String queue)
+ throws YarnException {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue);
+
+ GetSubClusterPolicyConfigurationResponse result =
+ stateStore.getPolicyConfiguration(request);
+ return result.getPolicyConfiguration();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9abcba9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
index 9396eda..74404c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
-import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
/**
* Unit tests for MemoryFederationStateStore.
@@ -26,7 +26,7 @@ public class TestMemoryFederationStateStore
extends FederationStateStoreBaseTest {
@Override
- protected FederationMembershipStateStore getCleanStateStore() {
+ protected FederationStateStore createStateStore() {
return new MemoryFederationStateStore();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-5406. In-memory based
implementation of the FederationMembershipStateStore. Contributed by Ellen
Hui.
Posted by su...@apache.org.
YARN-5406. In-memory based implementation of the FederationMembershipStateStore. Contributed by Ellen Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5861f61b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5861f61b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5861f61b
Branch: refs/heads/YARN-2915
Commit: 5861f61b1e5baf63f66041dc1d14c19ea7662d05
Parents: e2b5915
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Aug 4 15:54:38 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../store/impl/MemoryFederationStateStore.java | 138 ++++++++++++
.../federation/store/impl/package-info.java | 17 ++
.../records/GetSubClustersInfoRequest.java | 4 +
.../store/records/SubClusterState.java | 4 +
.../impl/FederationStateStoreBaseTest.java | 221 +++++++++++++++++++
.../impl/TestMemoryFederationStateStore.java | 49 ++++
6 files changed, 433 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5861f61b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
new file mode 100644
index 0000000..7fdc4a9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * In-memory implementation of FederationMembershipStateStore.
+ */
+public class MemoryFederationStateStore
+ implements FederationMembershipStateStore {
+
+ private final Map<SubClusterId, SubClusterInfo> membership =
+ new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
+ private final MonotonicClock clock = new MonotonicClock();
+
+ @Override
+ public Version getMembershipStateStoreVersion() {
+ return null;
+ }
+
+ @Override
+ public SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest request) throws YarnException {
+ SubClusterInfo subClusterInfo = request.getSubClusterInfo();
+ subClusterInfo.setLastStartTime(clock.getTime());
+ membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
+ return SubClusterRegisterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest request) throws YarnException {
+ SubClusterInfo subClusterInfo = membership.get(request.getSubClusterId());
+ if (subClusterInfo == null) {
+ throw new YarnException(
+ "SubCluster " + request.getSubClusterId().toString() + " not found");
+ } else {
+ subClusterInfo.setState(request.getState());
+ }
+
+ return SubClusterDeregisterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest request) throws YarnException {
+
+ SubClusterId subClusterId = request.getSubClusterId();
+ SubClusterInfo subClusterInfo = membership.get(subClusterId);
+
+ if (subClusterInfo == null) {
+ throw new YarnException("Subcluster " + subClusterId.toString()
+ + " does not exist; cannot heartbeat");
+ }
+
+ subClusterInfo.setLastHeartBeat(clock.getTime());
+ subClusterInfo.setState(request.getState());
+ subClusterInfo.setCapability(request.getCapability());
+
+ return SubClusterHeartbeatResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest request) throws YarnException {
+ SubClusterId subClusterId = request.getSubClusterId();
+ if (!membership.containsKey(subClusterId)) {
+ throw new YarnException(
+ "Subcluster " + subClusterId.toString() + " does not exist");
+ }
+
+ return GetSubClusterInfoResponse.newInstance(membership.get(subClusterId));
+ }
+
+ @Override
+ public GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest request) throws YarnException {
+ List<SubClusterInfo> result = new ArrayList<SubClusterInfo>();
+
+ for (SubClusterInfo info : membership.values()) {
+ if (!request.getFilterInactiveSubClusters()
+ || info.getState().isActive()) {
+ result.add(info);
+ }
+ }
+ return GetSubClustersInfoResponse.newInstance(result);
+
+ }
+
+ @VisibleForTesting
+ public Map<SubClusterId, SubClusterInfo> getMembershipTable() {
+ return membership;
+ }
+
+ @VisibleForTesting
+ public void clearMembershipTable() {
+ membership.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5861f61b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java
new file mode 100644
index 0000000..56e1274
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.impl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5861f61b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
index 3264d81..90d2f99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
@@ -26,6 +26,10 @@ import org.apache.hadoop.yarn.util.Records;
/**
* Request class to obtain information about all sub-clusters that are
* participating in federation.
+ *
+ * If filterInactiveSubClusters is set to true, only active sub-clusters will be
+ * returned; otherwise, all sub-clusters will be returned regardless of state.
+ * By default, filterInactiveSubClusters is true.
*/
@Private
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5861f61b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
index 22cec99..ff49aaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
@@ -53,6 +53,10 @@ public enum SubClusterState {
return (this != SC_RUNNING && this != SC_NEW);
}
+ public boolean isActive() {
+ return this == SC_RUNNING;
+ }
+
public boolean isFinal() {
return (this == SC_UNREGISTERED || this == SC_DECOMMISSIONED
|| this == SC_LOST);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5861f61b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
new file mode 100644
index 0000000..7eb1c86
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.io.IOException;
+
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Base class for FederationMembershipStateStore implementations.
+ */
+public abstract class FederationStateStoreBaseTest {
+
+ static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreBaseTest.class);
+ private static final MonotonicClock CLOCK = new MonotonicClock();
+
+ private FederationMembershipStateStore stateStore = getStateStore();
+
+ @Before
+ public void before() throws IOException {
+ clearMembership();
+ }
+
+ @Test
+ public void testRegisterSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ SubClusterRegisterResponse result = stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ Map<SubClusterId, SubClusterInfo> membership = getMembership();
+
+ Assert.assertNotNull(membership.get(subClusterId));
+ Assert.assertNotNull(result);
+ Assert.assertEquals(subClusterInfo, membership.get(subClusterId));
+ }
+
+ @Test
+ public void testDeregisterSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+
+ SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
+
+ stateStore.deregisterSubCluster(deregisterRequest);
+
+ Map<SubClusterId, SubClusterInfo> membership = getMembership();
+ Assert.assertNotNull(membership.get(subClusterId));
+ Assert.assertEquals(membership.get(subClusterId).getState(),
+ SubClusterState.SC_UNREGISTERED);
+ }
+
+ @Test
+ public void testDeregisterSubClusterUnknownSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+
+ SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
+ try {
+ stateStore.deregisterSubCluster(deregisterRequest);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().startsWith("SubCluster SC not found"));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterInfo() throws Exception {
+
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ Assert.assertEquals(subClusterInfo,
+ stateStore.getSubCluster(request).getSubClusterInfo());
+ }
+
+ @Test
+ public void testGetSubClusterInfoUnknownSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+
+ try {
+ stateStore.getSubCluster(request).getSubClusterInfo();
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Subcluster SC does not exist"));
+ }
+ }
+
+ @Test
+ public void testGetAllSubClustersInfo() throws Exception {
+
+ SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+ SubClusterInfo subClusterInfo1 = createSubClusterInfo(subClusterId1);
+
+ SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+ SubClusterInfo subClusterInfo2 = createSubClusterInfo(subClusterId2);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo1));
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo2));
+
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
+ .newInstance(subClusterId1, SubClusterState.SC_RUNNING, ""));
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
+ .newInstance(subClusterId2, SubClusterState.SC_UNHEALTHY, ""));
+
+ Assert.assertTrue(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(true))
+ .getSubClusters().contains(subClusterInfo1));
+ Assert.assertFalse(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(true))
+ .getSubClusters().contains(subClusterInfo2));
+
+ Assert.assertTrue(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(false))
+ .getSubClusters().contains(subClusterInfo1));
+ Assert.assertTrue(
+ stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(false))
+ .getSubClusters().contains(subClusterInfo2));
+ }
+
+ @Test
+ public void testSubClusterHeartbeat() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+
+ SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+ stateStore.subClusterHeartbeat(heartbeatRequest);
+
+ Map<SubClusterId, SubClusterInfo> membership = getMembership();
+ Assert.assertEquals(membership.get(subClusterId).getState(),
+ SubClusterState.SC_RUNNING);
+ Assert.assertNotNull(membership.get(subClusterId).getLastHeartBeat());
+ }
+
+ @Test
+ public void testSubClusterHeartbeatUnknownSubCluster() throws Exception {
+ SubClusterId subClusterId = SubClusterId.newInstance("SC");
+ SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+
+ try {
+ stateStore.subClusterHeartbeat(heartbeatRequest);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Subcluster SC does not exist; cannot heartbeat"));
+ }
+ }
+
+ private SubClusterInfo createSubClusterInfo(SubClusterId subClusterId) {
+
+ String amRMAddress = "1.2.3.4:1";
+ String clientRMAddress = "1.2.3.4:2";
+ String rmAdminAddress = "1.2.3.4:3";
+ String webAppAddress = "1.2.3.4:4";
+
+ return SubClusterInfo.newInstance(subClusterId, amRMAddress,
+ clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
+ CLOCK.getTime(), "cabability");
+ }
+
+ protected abstract Map<SubClusterId, SubClusterInfo> getMembership();
+
+ protected abstract void clearMembership();
+
+ protected abstract FederationMembershipStateStore getStateStore();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5861f61b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
new file mode 100644
index 0000000..b74ffbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * Unit tests for MemoryFederationStateStore.
+ */
+public class TestMemoryFederationStateStore
+ extends FederationStateStoreBaseTest {
+
+ private static final MemoryFederationStateStore STATESTORE =
+ new MemoryFederationStateStore();
+
+ @Override
+ protected Map<SubClusterId, SubClusterInfo> getMembership() {
+ return STATESTORE.getMembershipTable();
+ }
+
+ @Override
+ protected void clearMembership() {
+ STATESTORE.clearMembershipTable();
+ }
+
+ @Override
+ protected FederationMembershipStateStore getStateStore() {
+ return STATESTORE;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-5325. Stateless ARMRMProxy
policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5325. Stateless ARMRMProxy policies implementation. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1793757d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1793757d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1793757d
Branch: refs/heads/YARN-2915
Commit: 1793757ddb07118d0c007b84db18b05b3cee25c6
Parents: 5f03e0f
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Oct 13 17:59:13 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../AbstractConfigurableFederationPolicy.java | 155 +++++
.../policies/ConfigurableFederationPolicy.java | 9 +-
.../FederationPolicyInitializationContext.java | 37 +-
...ionPolicyInitializationContextValidator.java | 28 +-
.../policies/FederationPolicyManager.java | 59 +-
.../amrmproxy/AbstractAMRMProxyPolicy.java | 47 ++
.../amrmproxy/BroadcastAMRMProxyPolicy.java | 85 +++
.../amrmproxy/FederationAMRMProxyPolicy.java | 25 +-
.../LocalityMulticastAMRMProxyPolicy.java | 583 +++++++++++++++++++
.../policies/amrmproxy/package-info.java | 1 -
.../policies/dao/WeightedPolicyInfo.java | 180 +++---
.../federation/policies/dao/package-info.java | 1 -
.../policies/exceptions/package-info.java | 1 -
.../federation/policies/package-info.java | 1 -
.../policies/router/AbstractRouterPolicy.java | 47 ++
.../router/BaseWeightedRouterPolicy.java | 150 -----
.../policies/router/FederationRouterPolicy.java | 5 +-
.../policies/router/LoadBasedRouterPolicy.java | 36 +-
.../policies/router/PriorityRouterPolicy.java | 19 +-
.../router/UniformRandomRouterPolicy.java | 28 +-
.../router/WeightedRandomRouterPolicy.java | 32 +-
.../policies/router/package-info.java | 1 -
.../resolver/AbstractSubClusterResolver.java | 4 +-
.../policies/BaseFederationPoliciesTest.java | 28 +-
...ionPolicyInitializationContextValidator.java | 25 +-
.../TestBroadcastAMRMProxyFederationPolicy.java | 112 ++++
.../TestLocalityMulticastAMRMProxyPolicy.java | 566 ++++++++++++++++++
.../router/TestLoadBasedRouterPolicy.java | 18 +-
.../router/TestPriorityRouterPolicy.java | 15 +-
.../router/TestWeightedRandomRouterPolicy.java | 35 +-
.../utils/FederationPoliciesTestUtil.java | 64 ++
.../src/test/resources/nodes | 6 +-
32 files changed, 1950 insertions(+), 453 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java
new file mode 100644
index 0000000..4cb9bbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * Base abstract class for a weighted {@link ConfigurableFederationPolicy}.
+ */
+public abstract class AbstractConfigurableFederationPolicy
+ implements ConfigurableFederationPolicy {
+
+ private WeightedPolicyInfo policyInfo = null;
+ private FederationPolicyInitializationContext policyContext;
+ private boolean isDirty;
+
+ public AbstractConfigurableFederationPolicy() {
+ }
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext initializationContext)
+ throws FederationPolicyInitializationException {
+ isDirty = true;
+ FederationPolicyInitializationContextValidator
+ .validate(initializationContext, this.getClass().getCanonicalName());
+
+ // perform consistency checks
+ WeightedPolicyInfo newPolicyInfo = WeightedPolicyInfo.fromByteBuffer(
+ initializationContext.getSubClusterPolicyConfiguration().getParams());
+
+ // if nothing has changed skip the rest of initialization
+ // and signal to childs that the reinit is free via isDirty var.
+ if (policyInfo != null && policyInfo.equals(newPolicyInfo)) {
+ isDirty = false;
+ return;
+ }
+
+ validate(newPolicyInfo);
+ setPolicyInfo(newPolicyInfo);
+ this.policyContext = initializationContext;
+ }
+
+ /**
+ * Overridable validation step for the policy configuration.
+ *
+ * @param newPolicyInfo the configuration to test.
+ *
+ * @throws FederationPolicyInitializationException if the configuration is not
+ * valid.
+ */
+ public void validate(WeightedPolicyInfo newPolicyInfo)
+ throws FederationPolicyInitializationException {
+ if (newPolicyInfo == null) {
+ throw new FederationPolicyInitializationException(
+ "The policy to " + "validate should not be null.");
+ }
+ }
+
+ /**
+ * Returns true whether the last reinitialization requires actual changes, or
+ * was "free" as the weights have not changed. This is used by subclasses
+ * overriding reinitialize and calling super.reinitialize() to know wheter to
+ * quit early.
+ *
+ * @return whether more work is needed to initialize.
+ */
+ public boolean getIsDirty() {
+ return isDirty;
+ }
+
+ /**
+ * Getter method for the configuration weights.
+ *
+ * @return the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public WeightedPolicyInfo getPolicyInfo() {
+ return policyInfo;
+ }
+
+ /**
+ * Setter method for the configuration weights.
+ *
+ * @param policyInfo the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public void setPolicyInfo(WeightedPolicyInfo policyInfo) {
+ this.policyInfo = policyInfo;
+ }
+
+ /**
+ * Getter method for the {@link FederationPolicyInitializationContext}.
+ *
+ * @return the context for this policy.
+ */
+ public FederationPolicyInitializationContext getPolicyContext() {
+ return policyContext;
+ }
+
+ /**
+ * Setter method for the {@link FederationPolicyInitializationContext}.
+ *
+ * @param policyContext the context to assign to this policy.
+ */
+ public void setPolicyContext(
+ FederationPolicyInitializationContext policyContext) {
+ this.policyContext = policyContext;
+ }
+
+ /**
+ * This methods gets active subclusters map from the {@code
+ * FederationStateStoreFacade} and validate it not being null/empty.
+ *
+ * @return the map of ids to info for all active subclusters.
+ *
+ * @throws YarnException if we can't get the list.
+ */
+ protected Map<SubClusterId, SubClusterInfo> getActiveSubclusters()
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getPolicyContext().getFederationStateStoreFacade().getSubClusters(true);
+
+ if (activeSubclusters == null || activeSubclusters.size() < 1) {
+ throw new NoActiveSubclustersException(
+ "Zero active subclusters, cannot pick where to send job.");
+ }
+ return activeSubclusters;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
index fd6ceea..5245772 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
@@ -31,14 +31,11 @@ public interface ConfigurableFederationPolicy {
* policies. The implementor should provide try-n-swap semantics, and retain
* state if possible.
*
- * @param federationPolicyInitializationContext the new context to provide to
- * implementor.
+ * @param policyContext the new context to provide to implementor.
*
* @throws FederationPolicyInitializationException in case the initialization
- * fails.
+ * fails.
*/
- void reinitialize(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext)
+ void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
index 9347fd0..46dd6eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.federation.policies;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
@@ -30,6 +31,7 @@ public class FederationPolicyInitializationContext {
private SubClusterPolicyConfiguration federationPolicyConfiguration;
private SubClusterResolver federationSubclusterResolver;
private FederationStateStoreFacade federationStateStoreFacade;
+ private SubClusterId homeSubcluster;
public FederationPolicyInitializationContext() {
federationPolicyConfiguration = null;
@@ -37,20 +39,19 @@ public class FederationPolicyInitializationContext {
federationStateStoreFacade = null;
}
- public FederationPolicyInitializationContext(SubClusterPolicyConfiguration
- policy, SubClusterResolver resolver, FederationStateStoreFacade
- storeFacade) {
+ public FederationPolicyInitializationContext(
+ SubClusterPolicyConfiguration policy, SubClusterResolver resolver,
+ FederationStateStoreFacade storeFacade) {
this.federationPolicyConfiguration = policy;
this.federationSubclusterResolver = resolver;
this.federationStateStoreFacade = storeFacade;
}
-
/**
* Getter for the {@link SubClusterPolicyConfiguration}.
*
* @return the {@link SubClusterPolicyConfiguration} to be used for
- * initialization.
+ * initialization.
*/
public SubClusterPolicyConfiguration getSubClusterPolicyConfiguration() {
return federationPolicyConfiguration;
@@ -59,8 +60,8 @@ public class FederationPolicyInitializationContext {
/**
* Setter for the {@link SubClusterPolicyConfiguration}.
*
- * @param fedPolicyConfiguration the {@link SubClusterPolicyConfiguration}
- * to be used for initialization.
+ * @param fedPolicyConfiguration the {@link SubClusterPolicyConfiguration} to
+ * be used for initialization.
*/
public void setSubClusterPolicyConfiguration(
SubClusterPolicyConfiguration fedPolicyConfiguration) {
@@ -80,7 +81,7 @@ public class FederationPolicyInitializationContext {
* Setter for the {@link SubClusterResolver}.
*
* @param federationSubclusterResolver the {@link SubClusterResolver} to be
- * used for initialization.
+ * used for initialization.
*/
public void setFederationSubclusterResolver(
SubClusterResolver federationSubclusterResolver) {
@@ -105,4 +106,24 @@ public class FederationPolicyInitializationContext {
FederationStateStoreFacade federationStateStoreFacade) {
this.federationStateStoreFacade = federationStateStoreFacade;
}
+
+ /**
+ * Returns the current home sub-cluster. Useful for default policy behaviors.
+ *
+ * @return the home sub-cluster.
+ */
+ public SubClusterId getHomeSubcluster() {
+ return homeSubcluster;
+ }
+
+ /**
+ * Sets in the context the home sub-cluster. Useful for default policy
+ * behaviors.
+ *
+ * @param homeSubcluster value to set.
+ */
+ public void setHomeSubcluster(SubClusterId homeSubcluster) {
+ this.homeSubcluster = homeSubcluster;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
index 31f83d4..1b83bbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
@@ -25,50 +25,44 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPo
public final class FederationPolicyInitializationContextValidator {
private FederationPolicyInitializationContextValidator() {
- //disable constructor per checkstyle
+ // disable constructor per checkstyle
}
public static void validate(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
- String myType) throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContext policyContext, String myType)
+ throws FederationPolicyInitializationException {
if (myType == null) {
- throw new FederationPolicyInitializationException("The myType parameter"
- + " should not be null.");
+ throw new FederationPolicyInitializationException(
+ "The myType parameter" + " should not be null.");
}
- if (federationPolicyInitializationContext == null) {
+ if (policyContext == null) {
throw new FederationPolicyInitializationException(
"The FederationPolicyInitializationContext provided is null. Cannot"
- + " reinitalize "
- + "successfully.");
+ + " reinitalize " + "successfully.");
}
- if (federationPolicyInitializationContext.getFederationStateStoreFacade()
- == null) {
+ if (policyContext.getFederationStateStoreFacade() == null) {
throw new FederationPolicyInitializationException(
"The FederationStateStoreFacade provided is null. Cannot"
+ " reinitalize successfully.");
}
- if (federationPolicyInitializationContext.getFederationSubclusterResolver()
- == null) {
+ if (policyContext.getFederationSubclusterResolver() == null) {
throw new FederationPolicyInitializationException(
"The FederationStateStoreFacase provided is null. Cannot"
+ " reinitalize successfully.");
}
- if (federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
- == null) {
+ if (policyContext.getSubClusterPolicyConfiguration() == null) {
throw new FederationPolicyInitializationException(
"The FederationSubclusterResolver provided is null. Cannot "
+ "reinitalize successfully.");
}
String intendedType =
- federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
- .getType();
+ policyContext.getSubClusterPolicyConfiguration().getType();
if (!myType.equals(intendedType)) {
throw new FederationPolicyInitializationException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
index e5dba63..39fdba3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
@@ -25,19 +25,19 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyCo
/**
*
* Implementors need to provide the ability to serliaze a policy and its
- * configuration as a {@link SubClusterPolicyConfiguration}, as well as
- * provide (re)initialization mechanics for the underlying
+ * configuration as a {@link SubClusterPolicyConfiguration}, as well as provide
+ * (re)initialization mechanics for the underlying
* {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
*
- * The serialization aspects are used by admin APIs or a policy engine to
- * store a serialized configuration in the {@code FederationStateStore},
- * while the getters methods are used to obtain a propertly inizialized
- * policy in the {@code Router} and {@code AMRMProxy} respectively.
+ * The serialization aspects are used by admin APIs or a policy engine to store
+ * a serialized configuration in the {@code FederationStateStore}, while the
+ * getters methods are used to obtain a propertly inizialized policy in the
+ * {@code Router} and {@code AMRMProxy} respectively.
*
- * This interface by design binds together
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} and
- * provide lifecycle support for serialization and deserialization, to reduce
- * configuration mistakes (combining incompatible policies).
+ * This interface by design binds together {@link FederationAMRMProxyPolicy} and
+ * {@link FederationRouterPolicy} and provide lifecycle support for
+ * serialization and deserialization, to reduce configuration mistakes
+ * (combining incompatible policies).
*
*/
public interface FederationPolicyManager {
@@ -50,23 +50,17 @@ public interface FederationPolicyManager {
* the implementors should attempt to reinitalize (retaining state). To affect
* a complete policy reset oldInstance should be null.
*
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
*
- * @return an updated {@link FederationAMRMProxyPolicy
- }.
+ * @return an updated {@link FederationAMRMProxyPolicy }.
*
* @throws FederationPolicyInitializationException if the initialization
- * cannot be completed
- * properly. The oldInstance
- * should be still valid in
- * case of failed
- * initialization.
+ * cannot be completed properly. The oldInstance should be still
+ * valid in case of failed initialization.
*/
FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationAMRMProxyPolicy oldInstance)
throws FederationPolicyInitializationException;
@@ -78,21 +72,17 @@ public interface FederationPolicyManager {
* implementors should attempt to reinitalize (retaining state). To affect a
* complete policy reset oldInstance shoulb be set to null.
*
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
*
* @return an updated {@link FederationRouterPolicy}.
*
* @throws FederationPolicyInitializationException if the initalization cannot
- * be completed properly. The
- * oldInstance should be still
- * valid in case of failed
- * initialization.
+ * be completed properly. The oldInstance should be still valid in
+ * case of failed initialization.
*/
FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationRouterPolicy oldInstance)
throws FederationPolicyInitializationException;
@@ -102,23 +92,24 @@ public interface FederationPolicyManager {
* store.
*
* @return a valid policy configuration representing this object
- * parametrization.
+ * parametrization.
*
* @throws FederationPolicyInitializationException if the current state cannot
- * be serialized properly
+ * be serialized properly
*/
SubClusterPolicyConfiguration serializeConf()
throws FederationPolicyInitializationException;
-
/**
* This method returns the queue this policy is configured for.
+ *
* @return the name of the queue.
*/
String getQueue();
/**
* This methods provides a setter for the queue this policy is specified for.
+ *
* @param queue the name of the queue.
*/
void setQueue(String queue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java
new file mode 100644
index 0000000..e853744
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/AbstractAMRMProxyPolicy.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+
+/**
+ * Base abstract class for {@link FederationAMRMProxyPolicy} implementations,
+ * that provides common validation for reinitialization.
+ */
+public abstract class AbstractAMRMProxyPolicy extends
+ AbstractConfigurableFederationPolicy implements FederationAMRMProxyPolicy {
+
+ @Override
+ public void validate(WeightedPolicyInfo newPolicyInfo)
+ throws FederationPolicyInitializationException {
+ super.validate(newPolicyInfo);
+ Map<SubClusterIdInfo, Float> newWeights =
+ newPolicyInfo.getAMRMPolicyWeights();
+ if (newWeights == null || newWeights.size() < 1) {
+ throw new FederationPolicyInitializationException(
+ "Weight vector cannot be null/empty.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
new file mode 100644
index 0000000..679f4d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.UnknownSubclusterException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} that simply
+ * broadcasts each {@link ResourceRequest} to all the available sub-clusters.
+ */
+public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ private Set<SubClusterId> knownClusterIds = new HashSet<>();
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+ // overrides initialize to avoid weight checks that do no apply for
+ // this policy.
+ FederationPolicyInitializationContextValidator
+ .validate(policyContext, this.getClass().getCanonicalName());
+ setPolicyContext(policyContext);
+ }
+
+ @Override
+ public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests) throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ Map<SubClusterId, List<ResourceRequest>> answer = new HashMap<>();
+
+ // simply broadcast the resource request to all sub-clusters
+ for (SubClusterId subClusterId : activeSubclusters.keySet()) {
+ answer.put(subClusterId, resourceRequests);
+ knownClusterIds.add(subClusterId);
+ }
+
+ return answer;
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+ if (!knownClusterIds.contains(subClusterId)) {
+ throw new UnknownSubclusterException(
+ "The response is received from a subcluster that is unknown to this "
+ + "policy.");
+ }
+ // stateless policy does not care about responses
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
index 4a3305c..0541df4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
@@ -17,18 +17,18 @@
package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import java.util.List;
-import java.util.Map;
-
/**
- * Implementors of this interface provide logic to split the list of {@link
- * ResourceRequest}s received by the AM among various RMs.
+ * Implementors of this interface provide logic to split the list of
+ * {@link ResourceRequest}s received by the AM among various RMs.
*/
public interface FederationAMRMProxyPolicy
extends ConfigurableFederationPolicy {
@@ -37,18 +37,17 @@ public interface FederationAMRMProxyPolicy
* Splits the {@link ResourceRequest}s from the client across one or more
* sub-clusters based on the policy semantics (e.g., broadcast, load-based).
*
- * @param resourceRequests the list of {@link ResourceRequest}s from the
- * AM to be split
+ * @param resourceRequests the list of {@link ResourceRequest}s from the AM to
+ * be split
*
* @return map of sub-cluster as identified by {@link SubClusterId} to the
- * list of {@link ResourceRequest}s that should be forwarded to it
+ * list of {@link ResourceRequest}s that should be forwarded to it
*
* @throws YarnException in case the request is malformed or no viable
- * sub-clusters can be found.
+ * sub-clusters can be found.
*/
Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
- List<ResourceRequest> resourceRequests)
- throws YarnException;
+ List<ResourceRequest> resourceRequests) throws YarnException;
/**
* This method should be invoked to notify the policy about responses being
@@ -60,7 +59,7 @@ public interface FederationAMRMProxyPolicy
*
* @throws YarnException in case the response is not valid
*/
- void notifyOfResponse(SubClusterId subClusterId,
- AllocateResponse response) throws YarnException;
+ void notifyOfResponse(SubClusterId subClusterId, AllocateResponse response)
+ throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
new file mode 100644
index 0000000..283f89e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -0,0 +1,583 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} interface that
+ * carefully multicasts the requests with the following behavior:
+ *
+ * <p>
+ * Host localized {@link ResourceRequest}s are always forwarded to the RM that
+ * owns the corresponding node, based on the feedback of a
+ * {@link SubClusterResolver}. If the {@link SubClusterResolver} cannot resolve
+ * this node we default to forwarding the {@link ResourceRequest} to the home
+ * sub-cluster.
+ * </p>
+ *
+ * <p>
+ * Rack localized {@link ResourceRequest}s are forwarded to the RMs that owns
+ * the corresponding rack. Note that in some deployments each rack could be
+ * striped across multiple RMs. Thsi policy respects that. If the
+ * {@link SubClusterResolver} cannot resolve this rack we default to forwarding
+ * the {@link ResourceRequest} to the home sub-cluster.
+ * </p>
+ *
+ * <p>
+ * ANY requests corresponding to node/rack local requests are forwarded only to
+ * the set of RMs that owns the corresponding localized requests. The number of
+ * containers listed in each ANY is proportional to the number of localized
+ * container requests (associated to this ANY via the same allocateRequestId).
+ * </p>
+ *
+ * <p>
+ * ANY that are not associated to node/rack local requests are split among RMs
+ * based on the "weights" in the {@link WeightedPolicyInfo} configuration *and*
+ * headroom information. The {@code headroomAlpha} parameter of the policy
+ * configuration indicates how much headroom contributes to the splitting
+ * choice. Value of 1.0f indicates the weights are interpreted only as 0/1
+ * boolean but all splitting is based on the advertised headroom (fallback to
+ * 1/N for RMs that we don't have headroom info from). An {@code headroomAlpha}
+ * value of 0.0f means headroom is ignored and all splitting decisions are
+ * proportional to the "weights" in the configuration of the policy.
+ * </p>
+ *
+ * <p>
+ * ANY of zero size are forwarded to all known subclusters (i.e., subclusters
+ * where we scheduled containers before), as they may represent a user attempt
+ * to cancel a previous request (and we are mostly stateless now, so should
+ * forward to all known RMs).
+ * </p>
+ *
+ * <p>
+ * Invariants:
+ * </p>
+ *
+ * <p>
+ * The policy always excludes non-active RMs.
+ * </p>
+ *
+ * <p>
+ * The policy always excludes RMs that do not appear in the policy configuration
+ * weights, or have a weight of 0 (even if localized resources explicit refer to
+ * it).
+ * </p>
+ *
+ * <p>
+ * (Bar rounding to closest ceiling of fractional containers) The sum of
+ * requests made to multiple RMs at the ANY level "adds-up" to the user request.
+ * The maximum possible excess in a given request is a number of containers less
+ * or equal to number of sub-clusters in the federation.
+ * </p>
+ */
+public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(LocalityMulticastAMRMProxyPolicy.class);
+
+ private Map<SubClusterId, Float> weights;
+ private SubClusterResolver resolver;
+
+ private Map<SubClusterId, Resource> headroom;
+ private float hrAlpha;
+ private FederationStateStoreFacade federationFacade;
+ private AllocationBookkeeper bookkeeper;
+ private SubClusterId homeSubcluster;
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+
+ // save reference to old weights
+ WeightedPolicyInfo tempPolicy = getPolicyInfo();
+
+ super.reinitialize(policyContext);
+ if (!getIsDirty()) {
+ return;
+ }
+
+ Map<SubClusterId, Float> newWeightsConverted = new HashMap<>();
+ boolean allInactive = true;
+ WeightedPolicyInfo policy = getPolicyInfo();
+ if (policy.getAMRMPolicyWeights() == null
+ || policy.getAMRMPolicyWeights().size() == 0) {
+ allInactive = false;
+ } else {
+ for (Map.Entry<SubClusterIdInfo, Float> e : policy.getAMRMPolicyWeights()
+ .entrySet()) {
+ if (e.getValue() > 0) {
+ allInactive = false;
+ }
+ newWeightsConverted.put(e.getKey().toId(), e.getValue());
+ }
+ }
+ if (allInactive) {
+ // reset the policyInfo and throw
+ setPolicyInfo(tempPolicy);
+ throw new FederationPolicyInitializationException(
+ "The weights used to configure "
+ + "this policy are all set to zero! (no ResourceRequest could be "
+ + "forwarded with this setting.)");
+ }
+
+ if (policyContext.getHomeSubcluster() == null) {
+ setPolicyInfo(tempPolicy);
+ throw new FederationPolicyInitializationException("The homeSubcluster "
+ + "filed in the context must be initialized to use this policy");
+ }
+
+ weights = newWeightsConverted;
+ resolver = policyContext.getFederationSubclusterResolver();
+
+ if (headroom == null) {
+ headroom = new ConcurrentHashMap<>();
+ }
+ hrAlpha = policy.getHeadroomAlpha();
+
+ this.federationFacade =
+ policyContext.getFederationStateStoreFacade();
+ this.bookkeeper = new AllocationBookkeeper();
+ this.homeSubcluster = policyContext.getHomeSubcluster();
+
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+ // stateless policy does not care about responses except tracking headroom
+ headroom.put(subClusterId, response.getAvailableResources());
+ }
+
+ @Override
+ public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests) throws YarnException {
+
+ // object used to accumulate statistics about the answer, initialize with
+ // active subclusters.
+ bookkeeper.reinitialize(federationFacade.getSubClusters(true));
+
+ List<ResourceRequest> nonLocalizedRequests =
+ new ArrayList<ResourceRequest>();
+
+ SubClusterId targetId = null;
+ Set<SubClusterId> targetIds = null;
+
+ // if the RR is resolved to a local subcluster add it directly (node and
+ // resolvable racks)
+ for (ResourceRequest rr : resourceRequests) {
+ targetId = null;
+ targetIds = null;
+
+ // Handle: ANY (accumulated for later)
+ if (ResourceRequest.isAnyLocation(rr.getResourceName())) {
+ nonLocalizedRequests.add(rr);
+ continue;
+ }
+
+ // Handle "node" requests
+ try {
+ targetId = resolver.getSubClusterForNode(rr.getResourceName());
+ } catch (YarnException e) {
+ // this might happen as we can't differentiate node from rack names
+ // we log altogether later
+ }
+ if (bookkeeper.isActiveAndEnabled(targetId)) {
+ bookkeeper.addLocalizedNodeRR(targetId, rr);
+ continue;
+ }
+
+ // Handle "rack" requests
+ try {
+ targetIds = resolver.getSubClustersForRack(rr.getResourceName());
+ } catch (YarnException e) {
+ // this might happen as we can't differentiate node from rack names
+ // we log altogether later
+ }
+ if (targetIds != null && targetIds.size() > 0) {
+ for (SubClusterId tid : targetIds) {
+ if (bookkeeper.isActiveAndEnabled(tid)) {
+ bookkeeper.addRackRR(tid, rr);
+ }
+ }
+ continue;
+ }
+
+ // Handle node/rack requests that the SubClusterResolver cannot map to
+ // any cluster. Defaulting to home subcluster.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ERROR resolving sub-cluster for resourceName: "
+ + rr.getResourceName() + " we are falling back to homeSubCluster:"
+ + homeSubcluster);
+ }
+
+ // If home-subcluster is not active, ignore node/rack request
+ if (bookkeeper.isActiveAndEnabled(homeSubcluster)) {
+ bookkeeper.addLocalizedNodeRR(homeSubcluster, rr);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("The homeSubCluster (" + homeSubcluster + ") we are "
+ + "defaulting to is not active, the ResourceRequest "
+ + "will be ignored.");
+ }
+ }
+ }
+
+ // handle all non-localized requests (ANY)
+ splitAnyRequests(nonLocalizedRequests, bookkeeper);
+
+ return bookkeeper.getAnswer();
+ }
+
+ /**
+ * It splits a list of non-localized resource requests among sub-clusters.
+ */
+ private void splitAnyRequests(List<ResourceRequest> originalResourceRequests,
+ AllocationBookkeeper allocationBookkeeper) throws YarnException {
+
+ for (ResourceRequest resourceRequest : originalResourceRequests) {
+
+ // FIRST: pick the target set of subclusters (based on whether this RR
+ // is associated with other localized requests via an allocationId)
+ Long allocationId = resourceRequest.getAllocationRequestId();
+ Set<SubClusterId> targetSubclusters;
+ if (allocationBookkeeper.getSubClustersForId(allocationId) != null) {
+ targetSubclusters =
+ allocationBookkeeper.getSubClustersForId(allocationId);
+ } else {
+ targetSubclusters = allocationBookkeeper.getActiveAndEnabledSC();
+ }
+
+ // SECOND: pick how much to ask to each RM for each request
+ splitIndividualAny(resourceRequest, targetSubclusters,
+ allocationBookkeeper);
+ }
+ }
+
+ /**
+ * Return a projection of this ANY {@link ResourceRequest} that belongs to
+ * this sub-cluster. This is done based on the "count" of the containers that
+ * require locality in each sublcuster (if any) or based on the "weights" and
+ * headroom.
+ */
+ private void splitIndividualAny(ResourceRequest originalResourceRequest,
+ Set<SubClusterId> targetSubclusters,
+ AllocationBookkeeper allocationBookkeeper) {
+
+ long allocationId = originalResourceRequest.getAllocationRequestId();
+
+ for (SubClusterId targetId : targetSubclusters) {
+ float numContainer = originalResourceRequest.getNumContainers();
+
+ // If the ANY request has 0 containers to begin with we must forward it to
+ // any RM we have previously contacted (this might be the user way
+ // to cancel a previous request).
+ if (numContainer == 0 && headroom.containsKey(targetId)) {
+ allocationBookkeeper.addAnyRR(targetId, originalResourceRequest);
+ }
+
+ // If ANY is associated with localized asks, split based on their ratio
+ if (allocationBookkeeper.getSubClustersForId(allocationId) != null) {
+ float localityBasedWeight = getLocalityBasedWeighting(allocationId,
+ targetId, allocationBookkeeper);
+ numContainer = numContainer * localityBasedWeight;
+ } else {
+ // split ANY based on load and policy configuration
+ float headroomWeighting =
+ getHeadroomWeighting(targetId, allocationBookkeeper);
+ float policyWeighting =
+ getPolicyConfigWeighting(targetId, allocationBookkeeper);
+ // hrAlpha controls how much headroom influencing decision
+ numContainer = numContainer
+ * (hrAlpha * headroomWeighting + (1 - hrAlpha) * policyWeighting);
+ }
+
+ // if the calculated request is non-empty add it to the answer
+ if (numContainer > 0) {
+ ResourceRequest out =
+ ResourceRequest.newInstance(originalResourceRequest.getPriority(),
+ originalResourceRequest.getResourceName(),
+ originalResourceRequest.getCapability(),
+ originalResourceRequest.getNumContainers(),
+ originalResourceRequest.getRelaxLocality(),
+ originalResourceRequest.getNodeLabelExpression(),
+ originalResourceRequest.getExecutionTypeRequest());
+ out.setAllocationRequestId(allocationId);
+ out.setNumContainers((int) Math.ceil(numContainer));
+ if (out.isAnyLocation(out.getResourceName())) {
+ allocationBookkeeper.addAnyRR(targetId, out);
+ } else {
+ allocationBookkeeper.addRackRR(targetId, out);
+ }
+ }
+ }
+ }
+
+ /**
+ * Compute the weight to assign to a subcluster based on how many local
+ * requests a subcluster is target of.
+ */
+ private float getLocalityBasedWeighting(long reqId, SubClusterId targetId,
+ AllocationBookkeeper allocationBookkeeper) {
+ float totWeight = allocationBookkeeper.getTotNumLocalizedContainers();
+ float localWeight =
+ allocationBookkeeper.getNumLocalizedContainers(reqId, targetId);
+ return totWeight > 0 ? localWeight / totWeight : 0;
+ }
+
+ /**
+ * Compute the "weighting" to give to a sublcuster based on the configured
+ * policy weights (for the active subclusters).
+ */
+ private float getPolicyConfigWeighting(SubClusterId targetId,
+ AllocationBookkeeper allocationBookkeeper) {
+ float totWeight = allocationBookkeeper.totPolicyWeight;
+ Float localWeight = weights.get(targetId);
+ return (localWeight != null && totWeight > 0) ? localWeight / totWeight : 0;
+ }
+
+ /**
+ * Compute the weighting based on available headroom. This is proportional to
+ * the available headroom memory announced by RM, or to 1/N for RMs we have
+ * not seen yet. If all RMs report zero headroom, we fallback to 1/N again.
+ */
+ private float getHeadroomWeighting(SubClusterId targetId,
+ AllocationBookkeeper allocationBookkeeper) {
+
+ // baseline weight for all RMs
+ float headroomWeighting =
+ 1 / (float) allocationBookkeeper.getActiveAndEnabledSC().size();
+
+ // if we have headroom infomration for this sub-cluster (and we are safe
+ // from /0 issues)
+ if (headroom.containsKey(targetId)
+ && allocationBookkeeper.totHeadroomMemory > 0) {
+ // compute which portion of the RMs that are active/enabled have reported
+ // their headroom (needed as adjustment factor)
+ // (note: getActiveAndEnabledSC should never be null/zero)
+ float ratioHeadroomKnown = allocationBookkeeper.totHeadRoomEnabledRMs
+ / (float) allocationBookkeeper.getActiveAndEnabledSC().size();
+
+ // headroomWeighting is the ratio of headroom memory in the targetId
+ // cluster / total memory. The ratioHeadroomKnown factor is applied to
+ // adjust for missing information and ensure sum of allocated containers
+ // closely approximate what the user asked (small excess).
+ headroomWeighting = (headroom.get(targetId).getMemorySize()
+ / allocationBookkeeper.totHeadroomMemory) * (ratioHeadroomKnown);
+ }
+ return headroomWeighting;
+ }
+
+ /**
+ * This helper class is used to book-keep the requests made to each
+ * subcluster, and maintain useful statistics to split ANY requests.
+ */
+ private final class AllocationBookkeeper {
+
+ // the answer being accumulated
+ private Map<SubClusterId, List<ResourceRequest>> answer = new TreeMap<>();
+
+ // stores how many containers we have allocated in each RM for localized
+ // asks, used to correctly "spread" the corresponding ANY
+ private Map<Long, Map<SubClusterId, AtomicLong>> countContainersPerRM =
+ new HashMap<>();
+
+ private Set<SubClusterId> activeAndEnabledSC = new HashSet<>();
+ private long totNumLocalizedContainers = 0;
+ private float totHeadroomMemory = 0;
+ private int totHeadRoomEnabledRMs = 0;
+ private float totPolicyWeight = 0;
+
+ private void reinitialize(
+ Map<SubClusterId, SubClusterInfo> activeSubclusters)
+ throws YarnException {
+
+ // reset data structures
+ answer.clear();
+ countContainersPerRM.clear();
+ activeAndEnabledSC.clear();
+ totNumLocalizedContainers = 0;
+ totHeadroomMemory = 0;
+ totHeadRoomEnabledRMs = 0;
+ totPolicyWeight = 0;
+
+ // pre-compute the set of subclusters that are both active and enabled by
+ // the policy weights, and accumulate their total weight
+ for (Map.Entry<SubClusterId, Float> entry : weights.entrySet()) {
+ if (entry.getValue() > 0
+ && activeSubclusters.containsKey(entry.getKey())) {
+ activeAndEnabledSC.add(entry.getKey());
+ totPolicyWeight += entry.getValue();
+ }
+ }
+
+ if (activeAndEnabledSC.size() < 1) {
+ throw new NoActiveSubclustersException(
+ "None of the subclusters enabled in this policy (weight>0) are "
+ + "currently active we cannot forward the ResourceRequest(s)");
+ }
+
+ // pre-compute headroom-based weights for active/enabled subclusters
+ for (Map.Entry<SubClusterId, Resource> r : headroom.entrySet()) {
+ if (activeAndEnabledSC.contains(r.getKey())) {
+ totHeadroomMemory += r.getValue().getMemorySize();
+ totHeadRoomEnabledRMs++;
+ }
+ }
+
+ }
+
+ /**
+ * Add to the answer a localized node request, and keeps track of statistics
+ * on a per-allocation-id and per-subcluster bases.
+ */
+ private void addLocalizedNodeRR(SubClusterId targetId, ResourceRequest rr) {
+ Preconditions.checkArgument(!rr.isAnyLocation(rr.getResourceName()));
+
+ if (!countContainersPerRM.containsKey(rr.getAllocationRequestId())) {
+ countContainersPerRM.put(rr.getAllocationRequestId(), new HashMap<>());
+ }
+ if (!countContainersPerRM.get(rr.getAllocationRequestId())
+ .containsKey(targetId)) {
+ countContainersPerRM.get(rr.getAllocationRequestId()).put(targetId,
+ new AtomicLong(0));
+ }
+ countContainersPerRM.get(rr.getAllocationRequestId()).get(targetId)
+ .addAndGet(rr.getNumContainers());
+
+ totNumLocalizedContainers += rr.getNumContainers();
+
+ internalAddToAnswer(targetId, rr);
+ }
+
+ /**
+ * Add a rack-local request to the final asnwer.
+ */
+ public void addRackRR(SubClusterId targetId, ResourceRequest rr) {
+ Preconditions.checkArgument(!rr.isAnyLocation(rr.getResourceName()));
+ internalAddToAnswer(targetId, rr);
+ }
+
+ /**
+ * Add an ANY request to the final answer.
+ */
+ private void addAnyRR(SubClusterId targetId, ResourceRequest rr) {
+ Preconditions.checkArgument(rr.isAnyLocation(rr.getResourceName()));
+ internalAddToAnswer(targetId, rr);
+ }
+
+ private void internalAddToAnswer(SubClusterId targetId,
+ ResourceRequest partialRR) {
+ if (!answer.containsKey(targetId)) {
+ answer.put(targetId, new ArrayList<ResourceRequest>());
+ }
+ answer.get(targetId).add(partialRR);
+ }
+
+ /**
+ * Return all known subclusters associated with an allocation id.
+ *
+ * @param allocationId the allocation id considered
+ *
+ * @return the list of {@link SubClusterId}s associated with this allocation
+ * id
+ */
+ private Set<SubClusterId> getSubClustersForId(long allocationId) {
+ if (countContainersPerRM.get(allocationId) == null) {
+ return null;
+ }
+ return countContainersPerRM.get(allocationId).keySet();
+ }
+
+ /**
+ * Return the answer accumulated so far.
+ *
+ * @return the answer
+ */
+ private Map<SubClusterId, List<ResourceRequest>> getAnswer() {
+ return answer;
+ }
+
+ /**
+ * Return the set of sub-clusters that are both active and allowed by our
+ * policy (weight > 0).
+ *
+ * @return a set of active and enabled {@link SubClusterId}s
+ */
+ private Set<SubClusterId> getActiveAndEnabledSC() {
+ return activeAndEnabledSC;
+ }
+
+ /**
+ * Return the total number of container coming from localized requests.
+ */
+ private long getTotNumLocalizedContainers() {
+ return totNumLocalizedContainers;
+ }
+
+ /**
+ * Returns the number of containers matching an allocation Id that are
+ * localized in the targetId subcluster.
+ */
+ private long getNumLocalizedContainers(long allocationId,
+ SubClusterId targetId) {
+ AtomicLong c = countContainersPerRM.get(allocationId).get(targetId);
+ return c == null ? 0 : c.get();
+ }
+
+ /**
+ * Returns true is the subcluster request is both active and enabled.
+ */
+ private boolean isActiveAndEnabled(SubClusterId targetId) {
+ if (targetId == null) {
+ return false;
+ } else {
+ return getActiveAndEnabledSC().contains(targetId);
+ }
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
index 99da20b..ef72647 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
@@ -17,4 +17,3 @@
*/
/** AMRMPRoxy policies. **/
package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
index a0fa37f..62eb03b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
@@ -17,10 +17,19 @@
package org.apache.hadoop.yarn.server.federation.policies.dao;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.api.json.JSONJAXBContext;
-import com.sun.jersey.api.json.JSONMarshaller;
-import com.sun.jersey.api.json.JSONUnmarshaller;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -29,24 +38,16 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.api.json.JSONUnmarshaller;
/**
* This is a DAO class for the configuration of parameteres for federation
* policies. This generalizes several possible configurations as two lists of
- * {@link SubClusterIdInfo} and corresponding weights as a
- * {@link Float}. The interpretation of the weight is left to the logic in
- * the policy.
+ * {@link SubClusterIdInfo} and corresponding weights as a {@link Float}. The
+ * interpretation of the weight is left to the logic in the policy.
*/
@InterfaceAudience.Private
@@ -57,12 +58,14 @@ public class WeightedPolicyInfo {
private static final Logger LOG =
LoggerFactory.getLogger(WeightedPolicyInfo.class);
-
+ private static JSONJAXBContext jsonjaxbContext = initContext();
private Map<SubClusterIdInfo, Float> routerPolicyWeights = new HashMap<>();
private Map<SubClusterIdInfo, Float> amrmPolicyWeights = new HashMap<>();
private float headroomAlpha;
- private static JSONJAXBContext jsonjaxbContext = initContext();
+ public WeightedPolicyInfo() {
+ // JAXB needs this
+ }
private static JSONJAXBContext initContext() {
try {
@@ -74,46 +77,6 @@ public class WeightedPolicyInfo {
return null;
}
- public WeightedPolicyInfo() {
- //JAXB needs this
- }
-
- /**
- * Setter method for Router weights.
- *
- * @param policyWeights the router weights.
- */
- public void setRouterPolicyWeights(
- Map<SubClusterIdInfo, Float> policyWeights) {
- this.routerPolicyWeights = policyWeights;
- }
-
- /**
- * Setter method for ARMRMProxy weights.
- *
- * @param policyWeights the amrmproxy weights.
- */
- public void setAMRMPolicyWeights(
- Map<SubClusterIdInfo, Float> policyWeights) {
- this.amrmPolicyWeights = policyWeights;
- }
-
- /**
- * Getter of the router weights.
- * @return the router weights.
- */
- public Map<SubClusterIdInfo, Float> getRouterPolicyWeights() {
- return routerPolicyWeights;
- }
-
- /**
- * Getter for AMRMProxy weights.
- * @return the AMRMProxy weights.
- */
- public Map<SubClusterIdInfo, Float> getAMRMPolicyWeights() {
- return amrmPolicyWeights;
- }
-
/**
* Deserializes a {@link WeightedPolicyInfo} from a byte UTF-8 JSON
* representation.
@@ -123,14 +86,14 @@ public class WeightedPolicyInfo {
* @return the {@link WeightedPolicyInfo} represented.
*
* @throws FederationPolicyInitializationException if a deserializaiton error
- * occurs.
+ * occurs.
*/
public static WeightedPolicyInfo fromByteBuffer(ByteBuffer bb)
throws FederationPolicyInitializationException {
if (jsonjaxbContext == null) {
- throw new FederationPolicyInitializationException("JSONJAXBContext should"
- + " not be null.");
+ throw new FederationPolicyInitializationException(
+ "JSONJAXBContext should" + " not be null.");
}
try {
@@ -139,9 +102,8 @@ public class WeightedPolicyInfo {
bb.get(bytes);
String params = new String(bytes, Charset.forName("UTF-8"));
- WeightedPolicyInfo weightedPolicyInfo = unmarshaller
- .unmarshalFromJSON(new StringReader(params),
- WeightedPolicyInfo.class);
+ WeightedPolicyInfo weightedPolicyInfo = unmarshaller.unmarshalFromJSON(
+ new StringReader(params), WeightedPolicyInfo.class);
return weightedPolicyInfo;
} catch (JAXBException j) {
throw new FederationPolicyInitializationException(j);
@@ -149,19 +111,56 @@ public class WeightedPolicyInfo {
}
/**
- * Converts the policy into a byte array representation in the input {@link
- * ByteBuffer}.
+ * Getter of the router weights.
+ *
+ * @return the router weights.
+ */
+ public Map<SubClusterIdInfo, Float> getRouterPolicyWeights() {
+ return routerPolicyWeights;
+ }
+
+ /**
+ * Setter method for Router weights.
+ *
+ * @param policyWeights the router weights.
+ */
+ public void setRouterPolicyWeights(
+ Map<SubClusterIdInfo, Float> policyWeights) {
+ this.routerPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Getter for AMRMProxy weights.
+ *
+ * @return the AMRMProxy weights.
+ */
+ public Map<SubClusterIdInfo, Float> getAMRMPolicyWeights() {
+ return amrmPolicyWeights;
+ }
+
+ /**
+ * Setter method for ARMRMProxy weights.
+ *
+ * @param policyWeights the amrmproxy weights.
+ */
+ public void setAMRMPolicyWeights(Map<SubClusterIdInfo, Float> policyWeights) {
+ this.amrmPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Converts the policy into a byte array representation in the input
+ * {@link ByteBuffer}.
*
* @return byte array representation of this policy configuration.
*
* @throws FederationPolicyInitializationException if a serialization error
- * occurs.
+ * occurs.
*/
public ByteBuffer toByteBuffer()
throws FederationPolicyInitializationException {
if (jsonjaxbContext == null) {
- throw new FederationPolicyInitializationException("JSONJAXBContext should"
- + " not be null.");
+ throw new FederationPolicyInitializationException(
+ "JSONJAXBContext should" + " not be null.");
}
try {
String s = toJSONString();
@@ -186,22 +185,21 @@ public class WeightedPolicyInfo {
return false;
}
- WeightedPolicyInfo otherPolicy =
- (WeightedPolicyInfo) other;
+ WeightedPolicyInfo otherPolicy = (WeightedPolicyInfo) other;
Map<SubClusterIdInfo, Float> otherAMRMWeights =
otherPolicy.getAMRMPolicyWeights();
Map<SubClusterIdInfo, Float> otherRouterWeights =
otherPolicy.getRouterPolicyWeights();
- boolean amrmWeightsMatch = otherAMRMWeights != null &&
- getAMRMPolicyWeights() != null &&
- CollectionUtils.isEqualCollection(otherAMRMWeights.entrySet(),
- getAMRMPolicyWeights().entrySet());
+ boolean amrmWeightsMatch =
+ otherAMRMWeights != null && getAMRMPolicyWeights() != null
+ && CollectionUtils.isEqualCollection(otherAMRMWeights.entrySet(),
+ getAMRMPolicyWeights().entrySet());
- boolean routerWeightsMatch = otherRouterWeights != null &&
- getRouterPolicyWeights() != null &&
- CollectionUtils.isEqualCollection(otherRouterWeights.entrySet(),
- getRouterPolicyWeights().entrySet());
+ boolean routerWeightsMatch =
+ otherRouterWeights != null && getRouterPolicyWeights() != null
+ && CollectionUtils.isEqualCollection(otherRouterWeights.entrySet(),
+ getRouterPolicyWeights().entrySet());
return amrmWeightsMatch && routerWeightsMatch;
}
@@ -215,10 +213,10 @@ public class WeightedPolicyInfo {
* Return the parameter headroomAlpha, used by policies that balance
* weight-based and load-based considerations in their decisions.
*
- * For policies that use this parameter, values close to 1 indicate that
- * most of the decision should be based on currently observed headroom from
- * various sub-clusters, values close to zero, indicate that the decision
- * should be mostly based on weights and practically ignore current load.
+ * For policies that use this parameter, values close to 1 indicate that most
+ * of the decision should be based on currently observed headroom from various
+ * sub-clusters, values close to zero, indicate that the decision should be
+ * mostly based on weights and practically ignore current load.
*
* @return the value of headroomAlpha.
*/
@@ -227,13 +225,13 @@ public class WeightedPolicyInfo {
}
/**
- * Set the parameter headroomAlpha, used by policies that balance
- * weight-based and load-based considerations in their decisions.
+ * Set the parameter headroomAlpha, used by policies that balance weight-based
+ * and load-based considerations in their decisions.
*
- * For policies that use this parameter, values close to 1 indicate that
- * most of the decision should be based on currently observed headroom from
- * various sub-clusters, values close to zero, indicate that the decision
- * should be mostly based on weights and practically ignore current load.
+ * For policies that use this parameter, values close to 1 indicate that most
+ * of the decision should be based on currently observed headroom from various
+ * sub-clusters, values close to zero, indicate that the decision should be
+ * mostly based on weights and practically ignore current load.
*
* @param headroomAlpha the value to use for balancing.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
index 43f5b83..c292e52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
@@ -17,4 +17,3 @@
*/
/** DAO objects for serializing/deserializing policy configurations. **/
package org.apache.hadoop.yarn.server.federation.policies.dao;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
index 3318da9..ad2d543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
@@ -17,4 +17,3 @@
*/
/** Exceptions for policies. **/
package org.apache.hadoop.yarn.server.federation.policies.exceptions;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
index 7d9a121..fa3fcc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
@@ -17,4 +17,3 @@
*/
/** Federation Policies. **/
package org.apache.hadoop.yarn.server.federation.policies;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
new file mode 100644
index 0000000..f49af1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+
+/**
+ * Base abstract class for {@link FederationRouterPolicy} implementations, that
+ * provides common validation for reinitialization.
+ */
+public abstract class AbstractRouterPolicy extends
+ AbstractConfigurableFederationPolicy implements FederationRouterPolicy {
+
+ @Override
+ public void validate(WeightedPolicyInfo newPolicyInfo)
+ throws FederationPolicyInitializationException {
+ super.validate(newPolicyInfo);
+ Map<SubClusterIdInfo, Float> newWeights =
+ newPolicyInfo.getRouterPolicyWeights();
+ if (newWeights == null || newWeights.size() < 1) {
+ throw new FederationPolicyInitializationException(
+ "Weight vector cannot be null/empty.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
deleted file mode 100644
index e888979..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies.router;
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-
-import java.util.Map;
-
-/**
- * Abstract class provides common validation of reinitialize(), for all
- * policies that are "weight-based".
- */
-public abstract class BaseWeightedRouterPolicy
- implements FederationRouterPolicy {
-
- private WeightedPolicyInfo policyInfo = null;
- private FederationPolicyInitializationContext policyContext;
-
- public BaseWeightedRouterPolicy() {
- }
-
- @Override
- public void reinitialize(FederationPolicyInitializationContext
- federationPolicyContext)
- throws FederationPolicyInitializationException {
- FederationPolicyInitializationContextValidator
- .validate(federationPolicyContext, this.getClass().getCanonicalName());
-
- // perform consistency checks
- WeightedPolicyInfo newPolicyInfo = WeightedPolicyInfo
- .fromByteBuffer(
- federationPolicyContext.getSubClusterPolicyConfiguration()
- .getParams());
-
- // if nothing has changed skip the rest of initialization
- if (policyInfo != null && policyInfo.equals(newPolicyInfo)) {
- return;
- }
-
- validate(newPolicyInfo);
- setPolicyInfo(newPolicyInfo);
- this.policyContext = federationPolicyContext;
- }
-
- /**
- * Overridable validation step for the policy configuration.
- * @param newPolicyInfo the configuration to test.
- * @throws FederationPolicyInitializationException if the configuration is
- * not valid.
- */
- public void validate(WeightedPolicyInfo newPolicyInfo) throws
- FederationPolicyInitializationException {
- if (newPolicyInfo == null) {
- throw new FederationPolicyInitializationException("The policy to "
- + "validate should not be null.");
- }
- Map<SubClusterIdInfo, Float> newWeights =
- newPolicyInfo.getRouterPolicyWeights();
- if (newWeights == null || newWeights.size() < 1) {
- throw new FederationPolicyInitializationException(
- "Weight vector cannot be null/empty.");
- }
- }
-
-
- /**
- * Getter method for the configuration weights.
- *
- * @return the {@link WeightedPolicyInfo} representing the policy
- * configuration.
- */
- public WeightedPolicyInfo getPolicyInfo() {
- return policyInfo;
- }
-
- /**
- * Setter method for the configuration weights.
- *
- * @param policyInfo the {@link WeightedPolicyInfo} representing the policy
- * configuration.
- */
- public void setPolicyInfo(
- WeightedPolicyInfo policyInfo) {
- this.policyInfo = policyInfo;
- }
-
- /**
- * Getter method for the {@link FederationPolicyInitializationContext}.
- * @return the context for this policy.
- */
- public FederationPolicyInitializationContext getPolicyContext() {
- return policyContext;
- }
-
- /**
- * Setter method for the {@link FederationPolicyInitializationContext}.
- * @param policyContext the context to assign to this policy.
- */
- public void setPolicyContext(
- FederationPolicyInitializationContext policyContext) {
- this.policyContext = policyContext;
- }
-
- /**
- * This methods gets active subclusters map from the {@code
- * FederationStateStoreFacade} and validate it not being null/empty.
- *
- * @return the map of ids to info for all active subclusters.
- * @throws YarnException if we can't get the list.
- */
- protected Map<SubClusterId, SubClusterInfo> getActiveSubclusters()
- throws YarnException {
-
- Map<SubClusterId, SubClusterInfo> activeSubclusters = getPolicyContext()
- .getFederationStateStoreFacade().getSubClusters(true);
-
- if (activeSubclusters == null || activeSubclusters.size() < 1) {
- throw new NoActiveSubclustersException(
- "Zero active subclusters, cannot pick where to send job.");
- }
- return activeSubclusters;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
index 42c86cc..90ea0a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
@@ -35,11 +35,10 @@ public interface FederationRouterPolicy extends ConfigurableFederationPolicy {
* @param appSubmissionContext the context for the app being submitted.
*
* @return the sub-cluster as identified by {@link SubClusterId} to route the
- * request to.
+ * request to.
*
* @throws YarnException if the policy cannot determine a viable subcluster.
*/
SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException;
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-5467. InputValidator for the
FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
new file mode 100644
index 0000000..13175ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
@@ -0,0 +1,1265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for FederationApplicationInputValidator,
+ * FederationMembershipInputValidator, and FederationPolicyInputValidator.
+ */
+public class TestFederationStateStoreInputValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationStateStoreInputValidator.class);
+
+ private static SubClusterId subClusterId;
+ private static String amRMServiceAddress;
+ private static String clientRMServiceAddress;
+ private static String rmAdminServiceAddress;
+ private static String rmWebServiceAddress;
+ private static int lastHeartBeat;
+ private static SubClusterState stateNew;
+ private static SubClusterState stateLost;
+ private static ApplicationId appId;
+ private static int lastStartTime;
+ private static String capability;
+ private static String queue;
+ private static String type;
+ private static ByteBuffer params;
+
+ private static SubClusterId subClusterIdInvalid;
+ private static SubClusterId subClusterIdNull;
+
+ private static int lastHeartBeatNegative;
+ private static int lastStartTimeNegative;
+
+ private static SubClusterState stateNull;
+ private static ApplicationId appIdNull;
+
+ private static String capabilityNull;
+ private static String capabilityEmpty;
+
+ private static String addressNull;
+ private static String addressEmpty;
+ private static String addressWrong;
+ private static String addressWrongPort;
+
+ private static String queueEmpty;
+ private static String queueNull;
+
+ private static String typeEmpty;
+ private static String typeNull;
+
+ @BeforeClass
+ public static void setUp() {
+ subClusterId = SubClusterId.newInstance("abc");
+ amRMServiceAddress = "localhost:8032";
+ clientRMServiceAddress = "localhost:8034";
+ rmAdminServiceAddress = "localhost:8031";
+ rmWebServiceAddress = "localhost:8088";
+ lastHeartBeat = 1000;
+ stateNew = SubClusterState.SC_NEW;
+ stateLost = SubClusterState.SC_LOST;
+ lastStartTime = 1000;
+ capability = "Memory VCores";
+ appId = ApplicationId.newInstance(lastStartTime, 1);
+ queue = "default";
+ type = "random";
+ params = ByteBuffer.allocate(10);
+ params.put((byte) 0xFF);
+
+ subClusterIdInvalid = SubClusterId.newInstance("");
+ subClusterIdNull = null;
+
+ lastHeartBeatNegative = -10;
+ lastStartTimeNegative = -10;
+
+ stateNull = null;
+ appIdNull = null;
+
+ capabilityNull = null;
+ capabilityEmpty = "";
+
+ addressNull = null;
+ addressEmpty = "";
+ addressWrong = "AddressWrong";
+ addressWrongPort = "Address:WrongPort";
+
+ queueEmpty = "";
+ queueNull = null;
+
+ typeEmpty = "";
+ typeNull = null;
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequest() {
+
+ // Execution with valid inputs
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SubClusterRegisterRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterRegister Request."));
+ }
+
+ // Execution with null SubClusterInfo
+
+ subClusterInfo = null;
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Information."));
+ }
+
+ // Execution with Null SubClusterId
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterIdNull, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with Invalid SubClusterId
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterIdInvalid, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with Null State
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster State information."));
+ }
+
+ // Execution with Null Capability
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capabilityNull);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+
+ // Execution with Empty Capability
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capabilityEmpty);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequestTimestamp() {
+
+ // Execution with Negative Last Heartbeat
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeatNegative, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid timestamp information."));
+ }
+
+ // Execution with Negative Last StartTime
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTimeNegative, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid timestamp information."));
+ }
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequestAddress() {
+ // Execution with Null Address for amRMServiceAddress
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, addressNull,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for amRMServiceAddress
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId, addressEmpty,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Null Address for clientRMServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressNull, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for clientRMServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressEmpty, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Null Address for rmAdminServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressNull, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for rmAdminServiceAddress
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressEmpty, rmWebServiceAddress,
+ lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Null Address for rmWebServiceAddress
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressNull, lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+
+ // Execution with Empty Address for rmWebServiceAddress
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressEmpty, lastHeartBeat, stateNew, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SubCluster Endpoint information."));
+ }
+ }
+
+ @Test
+ public void testValidateSubClusterRegisterRequestAddressInvalid() {
+
+ // Address is not in host:port format for amRMService
+
+ SubClusterInfo subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, addressWrong,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Address is not in host:port format for clientRMService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressWrong, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Address is not in host:port format for rmAdminService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressWrong, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Address is not in host:port format for rmWebService
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressWrong, lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for amRMService
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId, addressWrongPort,
+ clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for clientRMService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ addressWrongPort, rmAdminServiceAddress, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for rmAdminService
+
+ subClusterInfo =
+ SubClusterInfo.newInstance(subClusterId, amRMServiceAddress,
+ clientRMServiceAddress, addressWrongPort, rmWebServiceAddress,
+ lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ // Port is not an integer for rmWebService
+
+ subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress,
+ addressWrongPort, lastHeartBeat, stateNull, lastStartTime, capability);
+ try {
+ SubClusterRegisterRequest request =
+ SubClusterRegisterRequest.newInstance(subClusterInfo);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("valid host:port authority:"));
+ }
+
+ }
+
+ @Test
+ public void testValidateSubClusterDeregisterRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterId, stateLost);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SubClusterDeregisterRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterDeregister Request."));
+ }
+
+ // Execution with null SubClusterId
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterIdNull, stateLost);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ try {
+ SubClusterDeregisterRequest request = SubClusterDeregisterRequest
+ .newInstance(subClusterIdInvalid, stateLost);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with null SubClusterState
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterId, stateNull);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster State information."));
+ }
+
+ // Execution with invalid SubClusterState
+
+ try {
+ SubClusterDeregisterRequest request =
+ SubClusterDeregisterRequest.newInstance(subClusterId, stateNew);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().startsWith("Invalid non-final state: "));
+ }
+ }
+
+ @Test
+ public void testSubClusterHeartbeatRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SubClusterHeartbeatRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterHeartbeat Request."));
+ }
+
+ // Execution with null SubClusterId
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterIdNull, lastHeartBeat, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ try {
+ SubClusterHeartbeatRequest request =
+ SubClusterHeartbeatRequest.newInstance(subClusterIdInvalid,
+ lastHeartBeat, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with null SubClusterState
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateNull, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster State information."));
+ }
+
+ // Execution with negative Last Heartbeat
+
+ try {
+ SubClusterHeartbeatRequest request =
+ SubClusterHeartbeatRequest.newInstance(subClusterId,
+ lastHeartBeatNegative, stateLost, capability);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid timestamp information."));
+ }
+
+ // Execution with null Capability
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateLost, capabilityNull);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+
+ // Execution with empty Capability
+
+ try {
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, lastHeartBeat, stateLost, capabilityEmpty);
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid capability information."));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterInfoRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ GetSubClusterInfoRequest request = null;
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing GetSubClusterInfo Request."));
+ }
+
+ // Execution with null SubClusterId
+
+ try {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterIdNull);
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ try {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterIdInvalid);
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+ }
+
+ @Test
+ public void testAddApplicationHomeSubClusterRequest() {
+
+ // Execution with valid inputs
+
+ ApplicationHomeSubCluster applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ AddApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing AddApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationHomeSubCluster
+
+ applicationHomeSubCluster = null;
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing ApplicationHomeSubCluster Info."));
+ }
+
+ // Execution with null SubClusterId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdNull);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdInvalid);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with Null ApplicationId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appIdNull, subClusterId);
+ try {
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterRequest() {
+
+ // Execution with valid inputs
+
+ ApplicationHomeSubCluster applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ UpdateApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing UpdateApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationHomeSubCluster
+
+ applicationHomeSubCluster = null;
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing ApplicationHomeSubCluster Info."));
+ }
+
+ // Execution with null SubClusteId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdNull);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubCluster Id information."));
+ }
+
+ // Execution with invalid SubClusterId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterIdInvalid);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ LOG.info(e.getMessage());
+ Assert.assertTrue(
+ e.getMessage().startsWith("Invalid SubCluster Id information."));
+ }
+
+ // Execution with null ApplicationId
+
+ applicationHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appIdNull, subClusterId);
+ try {
+ UpdateApplicationHomeSubClusterRequest request =
+ UpdateApplicationHomeSubClusterRequest
+ .newInstance(applicationHomeSubCluster);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ GetApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing GetApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationId
+
+ try {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appIdNull);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterRequestNull() {
+
+ // Execution with valid inputs
+
+ try {
+ DeleteApplicationHomeSubClusterRequest request =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ DeleteApplicationHomeSubClusterRequest request = null;
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing DeleteApplicationHomeSubCluster Request."));
+ }
+
+ // Execution with null ApplicationId
+
+ try {
+ DeleteApplicationHomeSubClusterRequest request =
+ DeleteApplicationHomeSubClusterRequest.newInstance(appIdNull);
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Application Id."));
+ }
+
+ }
+
+ @Test
+ public void testGetSubClusterPolicyConfigurationRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue);
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request = null;
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing GetSubClusterPolicyConfiguration Request."));
+ }
+
+ // Execution with null queue id
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queueNull);
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ // Execution with empty queue id
+
+ try {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queueEmpty);
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ }
+
+ @Test
+ public void testSetSubClusterPolicyConfigurationRequest() {
+
+ // Execution with valid inputs
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queue, type, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ // Execution with null request
+
+ try {
+ SetSubClusterPolicyConfigurationRequest request = null;
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage()
+ .startsWith("Missing SetSubClusterPolicyConfiguration Request."));
+ }
+
+ // Execution with null SubClusterPolicyConfiguration
+
+ try {
+ SubClusterPolicyConfiguration policy = null;
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Missing SubClusterPolicyConfiguration."));
+ }
+
+ // Execution with null queue id
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queueNull, type, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ // Execution with empty queue id
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queueEmpty, type, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Queue."));
+ }
+
+ // Execution with null policy type
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queue, typeNull, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Policy Type."));
+ }
+
+ // Execution with empty policy type
+
+ try {
+ SubClusterPolicyConfiguration policy =
+ SubClusterPolicyConfiguration.newInstance(queue, typeEmpty, params);
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest.newInstance(policy);
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
+ Assert.fail();
+ } catch (FederationStateStoreInvalidInputException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Missing Policy Type."));
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-5601. Make the RM epoch base
value configurable. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-5601. Make the RM epoch base value configurable. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4dd6de9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4dd6de9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4dd6de9
Branch: refs/heads/YARN-2915
Commit: d4dd6de9b8fe2f6f9beec86c8be6df1a9f2fd184
Parents: c55223e
Author: Jian He <ji...@apache.org>
Authored: Fri Sep 2 12:23:57 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 5 ++++-
.../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++
.../apache/hadoop/yarn/conf/TestYarnConfigurationFields.java | 2 ++
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 7 +++++++
.../resourcemanager/recovery/FileSystemRMStateStore.java | 2 +-
.../server/resourcemanager/recovery/LeveldbRMStateStore.java | 2 +-
.../server/resourcemanager/recovery/MemoryRMStateStore.java | 1 +
.../yarn/server/resourcemanager/recovery/RMStateStore.java | 4 ++++
.../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 2 +-
.../resourcemanager/recovery/RMStateStoreTestBase.java | 8 +++++---
.../server/resourcemanager/recovery/TestFSRMStateStore.java | 1 +
.../resourcemanager/recovery/TestLeveldbRMStateStore.java | 1 +
.../server/resourcemanager/recovery/TestZKRMStateStore.java | 1 +
13 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 2f5451d..bbd03a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -293,7 +293,10 @@
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore" />
- <Field name="resourceManager"/>
+ <Or>
+ <Field name="resourceManager"/>
+ <Field name="baseEpoch"/>
+ </Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7aa203d..2836e50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -142,6 +142,9 @@ public class YarnConfiguration extends Configuration {
public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
+ public static final String RM_EPOCH = RM_PREFIX + "epoch";
+ public static final long DEFAULT_RM_EPOCH = 0L;
+
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 5e0876f..3f3a06c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -75,6 +75,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.RM_EPOCH);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index e4bd64e..f84e5fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -781,6 +781,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
LOG.error("Failed to load/recover state", e);
throw e;
}
+ } else {
+ if (HAUtil.isFederationEnabled(conf)) {
+ long epoch = conf.getLong(YarnConfiguration.RM_EPOCH,
+ YarnConfiguration.DEFAULT_RM_EPOCH);
+ rmContext.setEpoch(epoch);
+ LOG.info("Epoch set for Federation: " + epoch);
+ }
}
super.serviceStart();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 929e2da..6468974 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -197,7 +197,7 @@ public class FileSystemRMStateStore extends RMStateStore {
@Override
public synchronized long getAndIncrementEpoch() throws Exception {
Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE);
- long currentEpoch = 0;
+ long currentEpoch = baseEpoch;
FileStatus status = getFileStatusWithRetries(epochNodePath);
if (status != null) {
// load current epoch
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index 02f90dd..dfe78e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -250,7 +250,7 @@ public class LeveldbRMStateStore extends RMStateStore {
@Override
public synchronized long getAndIncrementEpoch() throws Exception {
- long currentEpoch = 0;
+ long currentEpoch = baseEpoch;
byte[] dbKeyBytes = bytes(EPOCH_NODE);
try {
byte[] data = db.get(dbKeyBytes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
index 5f3328b..5041000 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
@@ -83,6 +83,7 @@ public class MemoryRMStateStore extends RMStateStore {
@Override
public synchronized void initInternal(Configuration conf) {
+ epoch = baseEpoch;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 5e3cf22..2a1948a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -99,6 +99,7 @@ public abstract class RMStateStore extends AbstractService {
"ReservationSystemRoot";
protected static final String VERSION_NODE = "RMVersionNode";
protected static final String EPOCH_NODE = "EpochNode";
+ protected long baseEpoch;
protected ResourceManager resourceManager;
private final ReadLock readLock;
private final WriteLock writeLock;
@@ -684,6 +685,9 @@ public abstract class RMStateStore extends AbstractService {
dispatcher.register(RMStateStoreEventType.class,
rmStateStoreEventHandler);
dispatcher.setDrainEventsOnStop();
+ // read the base epoch value from conf
+ baseEpoch = conf.getLong(YarnConfiguration.RM_EPOCH,
+ YarnConfiguration.DEFAULT_RM_EPOCH);
initInternal(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 1212a91..3cc7faa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -353,7 +353,7 @@ public class ZKRMStateStore extends RMStateStore {
@Override
public synchronized long getAndIncrementEpoch() throws Exception {
String epochNodePath = getNodePath(zkRootNodePath, EPOCH_NODE);
- long currentEpoch = 0;
+ long currentEpoch = baseEpoch;
if (exists(epochNodePath)) {
// load current epoch
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 514e9a0..771c16e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -91,6 +91,8 @@ public class RMStateStoreTestBase {
public static final Log LOG = LogFactory.getLog(RMStateStoreTestBase.class);
+ protected final long epoch = 10L;
+
static class TestDispatcher implements Dispatcher, EventHandler<Event> {
ApplicationAttemptId attemptId;
@@ -558,13 +560,13 @@ public class RMStateStoreTestBase {
store.setRMDispatcher(new TestDispatcher());
long firstTimeEpoch = store.getAndIncrementEpoch();
- Assert.assertEquals(0, firstTimeEpoch);
+ Assert.assertEquals(epoch, firstTimeEpoch);
long secondTimeEpoch = store.getAndIncrementEpoch();
- Assert.assertEquals(1, secondTimeEpoch);
+ Assert.assertEquals(epoch + 1, secondTimeEpoch);
long thirdTimeEpoch = store.getAndIncrementEpoch();
- Assert.assertEquals(2, thirdTimeEpoch);
+ Assert.assertEquals(epoch + 2, thirdTimeEpoch);
}
public void testAppDeletion(RMStateStoreHelper stateStoreHelper)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 6f0d53f..8c968cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -116,6 +116,7 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
900L);
+ conf.setLong(YarnConfiguration.RM_EPOCH, epoch);
if (adminCheckEnable) {
conf.setBoolean(
YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
index 4297e73..d6f77d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
@@ -81,6 +81,7 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase {
@Test(timeout = 60000)
public void testEpoch() throws Exception {
+ conf.setLong(YarnConfiguration.RM_EPOCH, epoch);
LeveldbStateStoreTester tester = new LeveldbStateStoreTester();
testEpoch(tester);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4dd6de9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index f71cf25..22eadea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -155,6 +155,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
conf.set(YarnConfiguration.RM_ZK_ADDRESS,
curatorTestingServer.getConnectString());
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
+ conf.setLong(YarnConfiguration.RM_EPOCH, epoch);
this.store = new TestZKRMStateStoreInternal(conf, workingZnode);
return this.store;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-5612. Return SubClusterId in
FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover.
(Giovanni Matteo Fumarola via Subru).
Posted by su...@apache.org.
YARN-5612. Return SubClusterId in FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover. (Giovanni Matteo Fumarola via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c55223e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c55223e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c55223e1
Branch: refs/heads/YARN-2915
Commit: c55223e1bd334622126dad1be3ea8d2189eebfe9
Parents: 23e6b16
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Sep 1 13:55:54 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../utils/FederationStateStoreFacade.java | 11 ++++---
.../utils/TestFederationStateStoreFacade.java | 30 ++++++++++++++++++++
2 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c55223e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index f1c8218..66a0b60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
@@ -298,13 +299,15 @@ public final class FederationStateStoreFacade {
*
* @param appHomeSubCluster the mapping of the application to it's home
* sub-cluster
+ * @return the stored Subcluster from StateStore
* @throws YarnException if the call to the state store is unsuccessful
*/
- public void addApplicationHomeSubCluster(
+ public SubClusterId addApplicationHomeSubCluster(
ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
- stateStore.addApplicationHomeSubCluster(
- AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
- return;
+ AddApplicationHomeSubClusterResponse response =
+ stateStore.addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+ return response.getHomeSubCluster();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c55223e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
index 53f4f84..d46bef0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
@@ -145,4 +146,33 @@ public class TestFederationStateStoreFacade {
}
}
+ @Test
+ public void testAddApplicationHomeSubCluster() throws YarnException {
+
+ // Inserting <AppId, Home1> into FederationStateStore
+ ApplicationId appId = ApplicationId.newInstance(clusterTs, numApps + 1);
+ SubClusterId subClusterId1 = SubClusterId.newInstance("Home1");
+
+ ApplicationHomeSubCluster appHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId1);
+
+ SubClusterId result =
+ facade.addApplicationHomeSubCluster(appHomeSubCluster);
+
+ Assert.assertEquals(facade.getApplicationHomeSubCluster(appId), result);
+ Assert.assertEquals(subClusterId1, result);
+
+ // Inserting <AppId, Home2> into FederationStateStore.
+ // The application is already present.
+ // FederationFacade will return Home1 as SubClusterId.
+ SubClusterId subClusterId2 = SubClusterId.newInstance("Home2");
+ appHomeSubCluster =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
+
+ result = facade.addApplicationHomeSubCluster(appHomeSubCluster);
+
+ Assert.assertEquals(facade.getApplicationHomeSubCluster(appId), result);
+ Assert.assertEquals(subClusterId1, result);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: MAPREDUCE-6825. YARNRunner#createApplicationSubmissionContext method is longer than 150 lines (Contributed by Gergely Novák via Daniel Templeton)
Posted by su...@apache.org.
MAPREDUCE-6825. YARNRunner#createApplicationSubmissionContext method is longer than 150 lines (Contributed by Gergely Nov�k via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/732ee6f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/732ee6f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/732ee6f0
Branch: refs/heads/YARN-2915
Commit: 732ee6f0b58a12500198c0d934cc570c7490b520
Parents: d150f06
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Feb 22 15:38:11 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Feb 22 15:38:11 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/YARNRunner.java | 141 +++++++++++--------
1 file changed, 86 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/732ee6f0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 98fe553..228c6af 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -291,8 +291,7 @@ public class YARNRunner implements ClientProtocol {
throws IOException, InterruptedException {
addHistoryToken(ts);
-
- // Construct necessary information to start the MR AM
+
ApplicationSubmissionContext appContext =
createApplicationSubmissionContext(conf, jobSubmitDir, ts);
@@ -331,34 +330,15 @@ public class YARNRunner implements ClientProtocol {
return rsrc;
}
- public ApplicationSubmissionContext createApplicationSubmissionContext(
- Configuration jobConf,
- String jobSubmitDir, Credentials ts) throws IOException {
- ApplicationId applicationId = resMgrDelegate.getApplicationId();
-
- // Setup resource requirements
- Resource capability = recordFactory.newRecordInstance(Resource.class);
- capability.setMemorySize(
- conf.getInt(
- MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
- )
- );
- capability.setVirtualCores(
- conf.getInt(
- MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
- )
- );
- LOG.debug("AppMaster capability = " + capability);
-
- // Setup LocalResources
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>();
+ private Map<String, LocalResource> setupLocalResources(Configuration jobConf,
+ String jobSubmitDir) throws IOException {
+ Map<String, LocalResource> localResources = new HashMap<>();
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
- URL yarnUrlForJobSubmitDir = URL.fromPath(defaultFileContext.getDefaultFileSystem()
- .resolvePath(
- defaultFileContext.makeQualified(new Path(jobSubmitDir))));
+ URL yarnUrlForJobSubmitDir = URL.fromPath(defaultFileContext
+ .getDefaultFileSystem().resolvePath(
+ defaultFileContext.makeQualified(new Path(jobSubmitDir))));
LOG.debug("Creating setup context, jobSubmitDir url is "
+ yarnUrlForJobSubmitDir);
@@ -371,7 +351,7 @@ public class YARNRunner implements ClientProtocol {
FileContext.getFileContext(jobJarPath.toUri(), jobConf),
jobJarPath,
LocalResourceType.PATTERN);
- String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+ String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
rc.setPattern(pattern);
localResources.put(MRJobConfig.JOB_JAR, rc);
@@ -392,13 +372,11 @@ public class YARNRunner implements ClientProtocol {
new Path(jobSubmitDir, s), LocalResourceType.FILE));
}
- // Setup security tokens
- DataOutputBuffer dob = new DataOutputBuffer();
- ts.writeTokenStorageToStream(dob);
- ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ return localResources;
+ }
- // Setup the command to run the AM
- List<String> vargs = new ArrayList<String>(8);
+ private List<String> setupAMCommand(Configuration jobConf) {
+ List<String> vargs = new ArrayList<>(8);
vargs.add(MRApps.crossPlatformifyMREnv(jobConf, Environment.JAVA_HOME)
+ "/bin/java");
@@ -409,27 +387,35 @@ public class YARNRunner implements ClientProtocol {
MRApps.addLog4jSystemProperties(null, vargs, conf);
// Check for Java Lib Path usage in MAP and REDUCE configs
- warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map",
- MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV);
- warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), "map",
- MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
- warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce",
- MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
- warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",
- MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS, ""),
+ "map",
+ MRJobConfig.MAP_JAVA_OPTS,
+ MRJobConfig.MAP_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, ""),
+ "map",
+ MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,
+ MRJobConfig.MAPRED_ADMIN_USER_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS, ""),
+ "reduce",
+ MRJobConfig.REDUCE_JAVA_OPTS,
+ MRJobConfig.REDUCE_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, ""),
+ "reduce",
+ MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,
+ MRJobConfig.MAPRED_ADMIN_USER_ENV);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
- warnForJavaLibPath(mrAppMasterAdminOptions, "app master",
+ warnForJavaLibPath(mrAppMasterAdminOptions, "app master",
MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV);
vargs.add(mrAppMasterAdminOptions);
-
+
// Add AM user command opts
String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
- warnForJavaLibPath(mrAppMasterUserOptions, "app master",
+ warnForJavaLibPath(mrAppMasterUserOptions, "app master",
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
@@ -449,9 +435,14 @@ public class YARNRunner implements ClientProtocol {
Path.SEPARATOR + ApplicationConstants.STDOUT);
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDERR);
+ return vargs;
+ }
+ private ContainerLaunchContext setupContainerLaunchContextForAM(
+ Configuration jobConf, Map<String, LocalResource> localResources,
+ ByteBuffer securityTokens, List<String> vargs) throws IOException {
- Vector<String> vargsFinal = new Vector<String>(8);
+ Vector<String> vargsFinal = new Vector<>(8);
// Final command
StringBuilder mergedCommand = new StringBuilder();
for (CharSequence str : vargs) {
@@ -464,7 +455,7 @@ public class YARNRunner implements ClientProtocol {
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
- Map<String, String> environment = new HashMap<String, String>();
+ Map<String, String> environment = new HashMap<>();
MRApps.setClasspath(environment, conf);
// Shell
@@ -477,28 +468,68 @@ public class YARNRunner implements ClientProtocol {
MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
// Setup the environment variables for Admin first
- MRApps.setEnvFromInputString(environment,
+ MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
- MRApps.setEnvFromInputString(environment,
+ MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ENV), conf);
// Parse distributed cache
MRApps.setupDistributedCache(jobConf, localResources);
- Map<ApplicationAccessType, String> acls
- = new HashMap<ApplicationAccessType, String>(2);
+ Map<ApplicationAccessType, String> acls = new HashMap<>(2);
acls.put(ApplicationAccessType.VIEW_APP, jobConf.get(
MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));
acls.put(ApplicationAccessType.MODIFY_APP, jobConf.get(
MRJobConfig.JOB_ACL_MODIFY_JOB,
MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB));
+ return ContainerLaunchContext.newInstance(localResources, environment,
+ vargsFinal, null, securityTokens, acls);
+ }
+
+ /**
+ * Constructs all the necessary information to start the MR AM.
+ * @param jobConf the configuration for the MR job
+ * @param jobSubmitDir the directory path for the job
+ * @param ts the security credentials for the job
+ * @return ApplicationSubmissionContext
+ * @throws IOException on IO error (e.g. path resolution)
+ */
+ public ApplicationSubmissionContext createApplicationSubmissionContext(
+ Configuration jobConf, String jobSubmitDir, Credentials ts)
+ throws IOException {
+ ApplicationId applicationId = resMgrDelegate.getApplicationId();
+
+ // Setup resource requirements
+ Resource capability = recordFactory.newRecordInstance(Resource.class);
+ capability.setMemorySize(
+ conf.getInt(
+ MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
+ )
+ );
+ capability.setVirtualCores(
+ conf.getInt(
+ MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
+ )
+ );
+ LOG.debug("AppMaster capability = " + capability);
+
+ // Setup LocalResources
+ Map<String, LocalResource> localResources =
+ setupLocalResources(jobConf, jobSubmitDir);
+
+ // Setup security tokens
+ DataOutputBuffer dob = new DataOutputBuffer();
+ ts.writeTokenStorageToStream(dob);
+ ByteBuffer securityTokens =
+ ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+
// Setup ContainerLaunchContext for AM container
- ContainerLaunchContext amContainer =
- ContainerLaunchContext.newInstance(localResources, environment,
- vargsFinal, null, securityTokens, acls);
+ List<String> vargs = setupAMCommand(jobConf);
+ ContainerLaunchContext amContainer = setupContainerLaunchContextForAM(
+ jobConf, localResources, securityTokens, vargs);
String regex = conf.get(MRJobConfig.MR_JOB_SEND_TOKEN_CONF);
if (regex != null && !regex.isEmpty()) {
@@ -566,7 +597,7 @@ public class YARNRunner implements ClientProtocol {
appContext.setApplicationType(MRJobConfig.MR_APPLICATION_TYPE);
if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
- appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
+ appContext.setApplicationTags(new HashSet<>(tagsFromConf));
}
String jobPriority = jobConf.get(MRJobConfig.PRIORITY);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-11426. Refactor EC CLI to be
similar to storage policies CLI.
Posted by su...@apache.org.
HDFS-11426. Refactor EC CLI to be similar to storage policies CLI.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132f758e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132f758e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132f758e
Branch: refs/heads/YARN-2915
Commit: 132f758e3dbe3a3f11c0d9b2de8edbee594fb475
Parents: 694e680
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Feb 23 16:00:11 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Feb 23 16:00:11 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/cli/CLITestHelper.java | 15 +-
.../hadoop-hdfs/src/main/bin/hdfs | 2 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 320 +++++++++++++++++++
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 2 +-
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 62 ----
.../hdfs/tools/erasurecode/ECCommand.java | 248 --------------
.../src/site/markdown/HDFSErasureCoding.md | 16 +-
.../hadoop/cli/CLITestCmdErasureCoding.java | 4 +-
.../cli/util/ErasureCodingCliCmdExecutor.java | 6 +-
.../test/resources/testErasureCodingConf.xml | 135 ++++----
10 files changed, 405 insertions(+), 405 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
index b08af16..89d4e30 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.cli;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.cli.util.*;
import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.conf.Configuration;
@@ -28,6 +26,9 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
@@ -41,9 +42,9 @@ import java.util.ArrayList;
* Tests for the Command Line Interface (CLI)
*/
public class CLITestHelper {
- private static final Log LOG =
- LogFactory.getLog(CLITestHelper.class.getName());
-
+ private static final Logger LOG = LoggerFactory.getLogger(CLITestHelper
+ .class);
+
// In this mode, it runs the command and compares the actual output
// with the expected output
public static final String TESTMODE_TEST = "test"; // Run the tests
@@ -62,7 +63,6 @@ public class CLITestHelper {
// Storage for tests read in from the config file
protected ArrayList<CLITestData> testsFromConfigFile = null;
protected ArrayList<ComparatorData> testComparators = null;
- protected String thisTestCaseName = null;
protected ComparatorData comparatorData = null;
protected Configuration conf = null;
protected String clitestDataDir = null;
@@ -80,7 +80,8 @@ public class CLITestHelper {
p.parse(testConfigFile, getConfigParser());
success = true;
} catch (Exception e) {
- LOG.info("File: " + testConfigFile + " not found");
+ LOG.info("Exception while reading test config file {}:",
+ testConfigFile, e);
success = false;
}
assertTrue("Error reading test config file", success);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 617adbe..cf6d94a 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -130,7 +130,7 @@ function hdfscmd_case
exit 0
;;
ec)
- HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.ECAdmin
;;
fetchdt)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
new file mode 100644
index 0000000..29c65b1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * CLI for the erasure code encoding operations.
+ */
+@InterfaceAudience.Private
+public class ECAdmin extends Configured implements Tool {
+
+ public static final String NAME = "ec";
+
+ public static void main(String[] args) throws Exception {
+ final ECAdmin admin = new ECAdmin(new Configuration());
+ int res = ToolRunner.run(admin, args);
+ System.exit(res);
+ }
+
+ public ECAdmin(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length == 0) {
+ AdminHelper.printUsage(false, NAME, COMMANDS);
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 1;
+ }
+ final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
+ COMMANDS);
+ if (command == null) {
+ System.err.println("Can't understand command '" + args[0] + "'");
+ if (!args[0].startsWith("-")) {
+ System.err.println("Command names must start with dashes.");
+ }
+ AdminHelper.printUsage(false, NAME, COMMANDS);
+ ToolRunner.printGenericCommandUsage(System.err);
+ return 1;
+ }
+ final List<String> argsList = new LinkedList<>();
+ argsList.addAll(Arrays.asList(args).subList(1, args.length));
+ try {
+ return command.run(getConf(), argsList);
+ } catch (IllegalArgumentException e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return -1;
+ }
+ }
+
+ /** Command to list the set of available erasure coding policies */
+ private static class ListECPoliciesCommand
+ implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-listPolicies";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + "]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ return getShortUsage() + "\n" +
+ "Get the list of supported erasure coding policies.\n";
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ try {
+ Collection<ErasureCodingPolicy> policies =
+ dfs.getAllErasureCodingPolicies();
+ System.out.println("Erasure Coding Policies:");
+ for (ErasureCodingPolicy policy : policies) {
+ if (policy != null) {
+ System.out.println("\t" + policy.getName());
+ }
+ }
+ } catch (IOException e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to get the erasure coding policy for a file or directory */
+ private static class GetECPolicyCommand implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-getPolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ final TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>",
+ "The path of the file/directory for getting the erasure coding " +
+ "policy");
+ return getShortUsage() + "\n" +
+ "Get the erasure coding policy of a file/directory.\n\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify the path with -path.\nUsage: " +
+ getLongUsage());
+ return 1;
+ }
+
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final Path p = new Path(path);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ try {
+ ErasureCodingPolicy ecPolicy = dfs.getErasureCodingPolicy(p);
+ if (ecPolicy != null) {
+ System.out.println(ecPolicy.getName());
+ } else {
+ System.out.println("The erasure coding policy of " + path + " is " +
+ "unspecified");
+ }
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to set the erasure coding policy to a file/directory */
+ private static class SetECPolicyCommand implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-setPolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path> -policy <policy>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>", "The path of the file/directory to set " +
+ "the erasure coding policy");
+ listing.addRow("<policy>", "The name of the erasure coding policy");
+ return getShortUsage() + "\n" +
+ "Set the erasure coding policy for a file/directory.\n\n" +
+ listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify the path for setting the EC " +
+ "policy.\nUsage: " + getLongUsage());
+ return 1;
+ }
+
+ final String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
+ args);
+ if (ecPolicyName == null) {
+ System.err.println("Please specify the policy name.\nUsage: " +
+ getLongUsage());
+ return 1;
+ }
+
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final Path p = new Path(path);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ try {
+ ErasureCodingPolicy ecPolicy = null;
+ ErasureCodingPolicy[] ecPolicies =
+ dfs.getClient().getErasureCodingPolicies();
+ for (ErasureCodingPolicy policy : ecPolicies) {
+ if (ecPolicyName.equals(policy.getName())) {
+ ecPolicy = policy;
+ break;
+ }
+ }
+ if (ecPolicy == null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Policy '");
+ sb.append(ecPolicyName);
+ sb.append("' does not match any of the supported policies.");
+ sb.append(" Please select any one of ");
+ List<String> ecPolicyNames = new ArrayList<String>();
+ for (ErasureCodingPolicy policy : ecPolicies) {
+ ecPolicyNames.add(policy.getName());
+ }
+ sb.append(ecPolicyNames);
+ System.err.println(sb.toString());
+ return 3;
+ }
+ dfs.setErasureCodingPolicy(p, ecPolicy);
+ System.out.println("Set erasure coding policy " + ecPolicyName +
+ " on " + path);
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ /** Command to unset the erasure coding policy set for a file/directory */
+ private static class UnsetECPolicyCommand
+ implements AdminHelper.Command {
+
+ @Override
+ public String getName() {
+ return "-unsetPolicy";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " -path <path>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<path>", "The path of the directory "
+ + "from which the erasure coding policy will be unset.");
+ return getShortUsage() + "\n"
+ + "Unset the erasure coding policy for a directory.\n\n"
+ + listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ if (path == null) {
+ System.err.println("Please specify a path.\nUsage: " + getLongUsage());
+ return 1;
+ }
+
+ if (args.size() > 0) {
+ System.err.println(getName() + ": Too many arguments");
+ return 1;
+ }
+
+ final Path p = new Path(path);
+ final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ try {
+ dfs.unsetErasureCodingPolicy(p);
+ System.out.println("Unset erasure coding policy from " + path);
+ } catch (Exception e) {
+ System.err.println(AdminHelper.prettifyException(e));
+ return 2;
+ }
+ return 0;
+ }
+ }
+
+ private static final AdminHelper.Command[] COMMANDS = {
+ new ListECPoliciesCommand(),
+ new GetECPolicyCommand(),
+ new SetECPolicyCommand(),
+ new UnsetECPolicyCommand()
+ };
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index f0643b2..9c7d048 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -143,7 +143,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
public int run(Configuration conf, List<String> args) throws IOException {
final String path = StringUtils.popOptionWithArgument("-path", args);
if (path == null) {
- System.err.println("Please specify the path with -path.\nUsage:" +
+ System.err.println("Please specify the path with -path.\nUsage: " +
getLongUsage());
return 1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
deleted file mode 100644
index 89dd4ee..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdfs.tools.erasurecode;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.fs.shell.CommandFactory;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-
-/**
- * CLI for the erasure code encoding operations.
- */
-@InterfaceAudience.Private
-public class ECCli extends FsShell {
-
- private final static String usagePrefix =
- "Usage: hdfs ec [generic options]";
-
- @Override
- protected String getUsagePrefix() {
- return usagePrefix;
- }
-
- @Override
- protected void init() throws IOException {
- getConf().setQuietMode(true);
- if (commandFactory == null) {
- commandFactory = new CommandFactory(getConf());
- commandFactory.addObject(getHelp(), "-help");
- registerCommands(commandFactory);
- }
- }
-
- @Override
- protected void registerCommands(CommandFactory factory) {
- factory.registerCommands(ECCommand.class);
- }
-
- public static void main(String[] args) throws Exception {
- Configuration conf = new HdfsConfiguration();
- int res = ToolRunner.run(conf, new ECCli(), args);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
deleted file mode 100644
index fc732e0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdfs.tools.erasurecode;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.shell.Command;
-import org.apache.hadoop.fs.shell.CommandFactory;
-import org.apache.hadoop.fs.shell.PathData;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.util.StringUtils;
-
-/**
- * Erasure Coding CLI commands
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class ECCommand extends Command {
-
- public static void registerCommands(CommandFactory factory) {
- // Register all commands of Erasure CLI, with a '-' at the beginning in name
- // of the command.
- factory.addClass(SetECPolicyCommand.class, "-" + SetECPolicyCommand.NAME);
- factory.addClass(GetECPolicyCommand.class, "-"
- + GetECPolicyCommand.NAME);
- factory.addClass(UnsetECPolicyCommand.class, "-"
- + UnsetECPolicyCommand.NAME);
- factory.addClass(ListPolicies.class, "-" + ListPolicies.NAME);
- }
-
- @Override
- public String getCommandName() {
- return getName();
- }
-
- @Override
- protected void run(Path path) throws IOException {
- throw new RuntimeException("Not suppose to get here");
- }
-
- @Deprecated
- @Override
- public int runAll() {
- return run(args);
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- if (!(item.fs instanceof DistributedFileSystem)) {
- throw new UnsupportedActionException(
- "Erasure commands are only supported for the HDFS paths");
- }
- }
-
- /**
- * A command to set the erasure coding policy for a directory, with the name
- * of the policy.
- */
- static class SetECPolicyCommand extends ECCommand {
- public static final String NAME = "setPolicy";
- public static final String USAGE = "[-p <policyName>] <path>";
- public static final String DESCRIPTION =
- "Set a specified erasure coding policy to a directory\n"
- + "Options :\n"
- + " -p <policyName> : erasure coding policy name to encode files. "
- + "If not passed the default policy will be used\n"
- + " <path> : Path to a directory. Under this directory "
- + "files will be encoded using specified erasure coding policy";
- private String ecPolicyName;
- private ErasureCodingPolicy ecPolicy = null;
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- ecPolicyName = StringUtils.popOptionWithArgument("-p", args);
- if (args.isEmpty()) {
- throw new HadoopIllegalArgumentException("<path> is missing");
- }
- if (args.size() > 1) {
- throw new HadoopIllegalArgumentException("Too many arguments");
- }
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- super.processPath(item);
- DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
- try {
- if (ecPolicyName != null) {
- ErasureCodingPolicy[] ecPolicies = dfs.getClient().getErasureCodingPolicies();
- for (ErasureCodingPolicy ecPolicy : ecPolicies) {
- if (ecPolicyName.equals(ecPolicy.getName())) {
- this.ecPolicy = ecPolicy;
- break;
- }
- }
- if (ecPolicy == null) {
- StringBuilder sb = new StringBuilder();
- sb.append("Policy '");
- sb.append(ecPolicyName);
- sb.append("' does not match any of the supported policies.");
- sb.append(" Please select any one of ");
- List<String> ecPolicyNames = new ArrayList<String>();
- for (ErasureCodingPolicy ecPolicy : ecPolicies) {
- ecPolicyNames.add(ecPolicy.getName());
- }
- sb.append(ecPolicyNames);
- throw new HadoopIllegalArgumentException(sb.toString());
- }
- }
- dfs.setErasureCodingPolicy(item.path, ecPolicy);
- out.println("EC policy set successfully at " + item.path);
- } catch (IOException e) {
- throw new IOException("Unable to set EC policy for the path "
- + item.path + ". " + e.getMessage());
- }
- }
- }
-
- /**
- * Get the erasure coding policy of a file or directory
- */
- static class GetECPolicyCommand extends ECCommand {
- public static final String NAME = "getPolicy";
- public static final String USAGE = "<path>";
- public static final String DESCRIPTION =
- "Get erasure coding policy information about at specified path\n";
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- if (args.isEmpty()) {
- throw new HadoopIllegalArgumentException("<path> is missing");
- }
- if (args.size() > 1) {
- throw new HadoopIllegalArgumentException("Too many arguments");
- }
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- super.processPath(item);
- DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
- try {
- ErasureCodingPolicy ecPolicy = dfs.getErasureCodingPolicy(item.path);
- if (ecPolicy != null) {
- out.println(ecPolicy.toString());
- } else {
- out.println("Path " + item.path + " is not erasure coded.");
- }
- } catch (IOException e) {
- throw new IOException("Unable to get EC policy for the path "
- + item.path + ". " + e.getMessage());
- }
- }
- }
-
- /**
- * List all supported erasure coding policies
- */
- static class ListPolicies extends ECCommand {
- public static final String NAME = "listPolicies";
- public static final String USAGE = "";
- public static final String DESCRIPTION =
- "Get the list of erasure coding policies supported\n";
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- if (!args.isEmpty()) {
- throw new HadoopIllegalArgumentException("Too many parameters");
- }
-
- FileSystem fs = FileSystem.get(getConf());
- if (fs instanceof DistributedFileSystem == false) {
- throw new UnsupportedActionException(
- "Erasure commands are only supported for the HDFS");
- }
- DistributedFileSystem dfs = (DistributedFileSystem) fs;
-
- ErasureCodingPolicy[] ecPolicies = dfs.getClient().getErasureCodingPolicies();
- StringBuilder sb = new StringBuilder();
- int i = 0;
- while (i < ecPolicies.length) {
- ErasureCodingPolicy ecPolicy = ecPolicies[i];
- sb.append(ecPolicy.getName());
- i++;
- if (i < ecPolicies.length) {
- sb.append(", ");
- }
- }
- out.println(sb.toString());
- }
- }
-
- /**
- * Unset the erasure coding policy from a directory.
- */
- static class UnsetECPolicyCommand extends ECCommand {
- public static final String NAME = "unsetPolicy";
- public static final String USAGE = "<path>";
- public static final String DESCRIPTION =
- "Unset erasure coding policy from a directory\n";
-
- @Override
- protected void processOptions(LinkedList<String> args) throws IOException {
- if (args.isEmpty()) {
- throw new HadoopIllegalArgumentException("<path> is missing");
- }
- if (args.size() > 1) {
- throw new HadoopIllegalArgumentException("Too many arguments");
- }
- }
-
- @Override
- protected void processPath(PathData item) throws IOException {
- super.processPath(item);
- DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
- try {
- dfs.unsetErasureCodingPolicy(item.path);
- } catch (IOException e) {
- throw new IOException("Unable to unset EC policy from directory "
- + item.path + ". " + e.getMessage());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 6e4891f..0283e2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -119,30 +119,30 @@ Deployment
HDFS provides an `ec` subcommand to perform administrative commands related to erasure coding.
hdfs ec [generic options]
- [-setPolicy [-p <policyName>] <path>]
- [-getPolicy <path>]
- [-unsetPolicy <path>]
+ [-setPolicy -policy <policyName> -path <path>]
+ [-getPolicy -path <path>]
+ [-unsetPolicy -path <path>]
[-listPolicies]
[-usage [cmd ...]]
[-help [cmd ...]]
Below are the details about each command.
- * `[-setPolicy [-p <policyName>] <path>]`
+ * `[-setPolicy -policy <policyName> -path <path>]`
Sets an ErasureCoding policy on a directory at the specified path.
`path`: An directory in HDFS. This is a mandatory parameter. Setting a policy only affects newly created files, and does not affect existing files.
- `policyName`: The ErasureCoding policy to be used for files under this directory. This is an optional parameter, specified using \u2018-p\u2019 flag. If no policy is specified, the system default ErasureCodingPolicy will be used.
+ `policyName`: The ErasureCoding policy to be used for files under this directory.
- * `[-getPolicy <path>]`
+ * `[-getPolicy -path <path>]`
Get details of the ErasureCoding policy of a file or directory at the specified path.
- * `[-unsetPolicy <path>]`
+ * `[-unsetPolicy -path <path>]`
- Unset an ErasureCoding policy from a directory at the specified path when previously user sets the ErasureCoding policy on this directory via "setPolicy" command. If the directory inherits the ErasureCoding policy from its parent group, "unsetPolicy" command on this directory will not have any effect. Unset ErasureCoding policy on a directory which doesn't have ErasureCoding policy will not return an error.
+ Unset an ErasureCoding policy set by a previous call to "setPolicy" on a directory. If the directory inherits the ErasureCoding policy from an ancestor directory, "unsetPolicy" is a no-op. Unsetting the policy on a directory which doesn't have an explicit policy set will not return an error.
* `[-listPolicies]`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
index 0499a2b..28e5f98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.cli.util.CLITestCmd;
import org.apache.hadoop.cli.util.CommandExecutor;
import org.apache.hadoop.cli.util.ErasureCodingCliCmdExecutor;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
+import org.apache.hadoop.hdfs.tools.ECAdmin;
public class CLITestCmdErasureCoding extends CLITestCmd {
public CLITestCmdErasureCoding(String str, CLICommandTypes type) {
@@ -33,7 +33,7 @@ public class CLITestCmdErasureCoding extends CLITestCmd {
@Override
public CommandExecutor getExecutor(String tag, Configuration conf) throws IllegalArgumentException {
if (getType() instanceof CLICommandErasureCodingCli)
- return new ErasureCodingCliCmdExecutor(tag, new ECCli());
+ return new ErasureCodingCliCmdExecutor(tag, new ECAdmin(conf));
return super.getExecutor(tag, conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
index e993313..59b2a73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
@@ -17,14 +17,14 @@
*/
package org.apache.hadoop.cli.util;
-import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
+import org.apache.hadoop.hdfs.tools.ECAdmin;
import org.apache.hadoop.util.ToolRunner;
public class ErasureCodingCliCmdExecutor extends CommandExecutor {
protected String namenode = null;
- protected ECCli admin = null;
+ protected ECAdmin admin = null;
- public ErasureCodingCliCmdExecutor(String namenode, ECCli admin) {
+ public ErasureCodingCliCmdExecutor(String namenode, ECAdmin admin) {
this.namenode = namenode;
this.admin = admin;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/132f758e/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 0753d4d..82b71def 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -42,7 +42,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>-usage: Unknown command</expected-output>
+ <expected-output>Can't understand command '-usage'</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Usage: bin/hdfs ec [COMMAND]</expected-output>
</comparator>
</comparators>
</test>
@@ -58,7 +62,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Usage: hdfs ec [generic options]</expected-output>
+ <expected-output>[-listPolicies]</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>[-unsetPolicy -path</expected-output>
</comparator>
</comparators>
</test>
@@ -72,12 +80,12 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^[ \t]*Set a specified erasure coding policy to a directory( )*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Set the erasure coding policy for a file/directory.</expected-output>
</comparator>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setPolicy \[-p <policyName>\] <path>(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>[-setPolicy -path <path> -policy <policy>]</expected-output>
</comparator>
</comparators>
</test>
@@ -92,11 +100,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Get erasure coding policy information about at specified path</expected-output>
+ <expected-output>Get the erasure coding policy of a file/directory</expected-output>
</comparator>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-getPolicy <path>(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>[-getPolicy -path <path>]</expected-output>
</comparator>
</comparators>
</test>
@@ -111,11 +119,11 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Get the list of erasure coding policies supported</expected-output>
+ <expected-output>Get the list of supported erasure coding policies</expected-output>
</comparator>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-listPolicies (.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>[-listPolicies]</expected-output>
</comparator>
</comparators>
</test>
@@ -125,7 +133,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -133,7 +141,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>EC policy set successfully at NAMENODE/ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -142,8 +150,8 @@
<description>setPolicy : set a policy twice</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -151,25 +159,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>EC policy set successfully at NAMENODE/ecdir</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test>
- <description>setPolicy : default policy</description>
- <test-commands>
- <command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
- </test-commands>
- <cleanup-commands>
- <command>-fs NAMENODE -rmdir /ecdir</command>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=</expected-output>
+ <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -178,9 +168,9 @@
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -unsetPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -188,7 +178,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>is not erasure coded.</expected-output>
+ <expected-output>is unspecified</expected-output>
</comparator>
</comparators>
</test>
@@ -197,10 +187,9 @@
<description>setPolicy : change different policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-3-2-64k
- /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -208,7 +197,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-3-2-64k</expected-output>
+ <expected-output>RS-DEFAULT-3-2-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -218,10 +207,10 @@
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -mkdir /ecdir/child</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -unsetPolicy /ecdir/child</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir/child/ecfile</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir/child/ecfile</command>
@@ -231,7 +220,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -240,7 +229,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /noec</command>
- <ec-admin-command>-fs NAMENODE -getPolicy /noec</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /noec</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /noec</command>
@@ -248,7 +237,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Path NAMENODE/noec is not erasure coded</expected-output>
+ <expected-output>is unspecified</expected-output>
</comparator>
</comparators>
</test>
@@ -257,8 +246,8 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -266,7 +255,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -275,9 +264,9 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir/ecfile</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir/ecfile</command>
@@ -286,7 +275,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -318,8 +307,8 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setPolicy: <path> is missing(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Please specify the path for setting the EC policy.</expected-output>
</comparator>
</comparators>
</test>
@@ -328,15 +317,15 @@
<description>setPolicy : illegal parameters - policy name is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setPolicy: option -p requires 1 argument(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>option -path requires 1 argument</expected-output>
</comparator>
</comparators>
</test>
@@ -345,7 +334,7 @@
<description>setPolicy : illegal parameters - too many arguments</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir1 /ecdir2</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -362,7 +351,7 @@
<description>setPolicy : illegal parameters - invalidpolicy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -p invalidpolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy invalidpolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -378,14 +367,14 @@
<test>
<description>setPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -setPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^setPolicy: `/ecdir': No such file or directory(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Path not found: /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -399,8 +388,8 @@
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^-getPolicy: <path> is missing(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Please specify the path with -path</expected-output>
</comparator>
</comparators>
</test>
@@ -408,7 +397,7 @@
<test>
<description>getPolicy : illegal parameters - too many arguments</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm /ecdir</command>
@@ -424,14 +413,14 @@
<test>
<description>getPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
<comparators>
<comparator>
- <type>RegexpComparator</type>
- <expected-output>^getPolicy: `/ecdir': No such file or directory(.)*</expected-output>
+ <type>SubstringComparator</type>
+ <expected-output>Path not found: /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -446,7 +435,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>-listPolicies: Too many parameters</expected-output>
+ <expected-output>-listPolicies: Too many arguments</expected-output>
</comparator>
</comparators>
</test>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-6222.
TestFairScheduler.testReservationMetrics is flaky. (Yufei Gu via kasha)
Posted by su...@apache.org.
YARN-6222. TestFairScheduler.testReservationMetrics is flaky. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/694e680d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/694e680d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/694e680d
Branch: refs/heads/YARN-2915
Commit: 694e680d20dc07f634b539537021b09d9316601c
Parents: 159d6c5
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Thu Feb 23 15:21:52 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Thu Feb 23 15:21:52 2017 -0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java | 3 +--
.../server/resourcemanager/scheduler/fair/TestFairScheduler.java | 1 +
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/694e680d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 59bde5b..d0e0961 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -407,8 +407,7 @@ public class FSLeafQueue extends FSQueue {
readLock.lock();
try {
for (FSAppAttempt app : runnableApps) {
- Resource pending = app.getAppAttemptResourceUsage().getPending();
- if (!Resources.isNone(pending) &&
+ if (!Resources.isNone(app.getPendingDemand()) &&
(assignment || app.shouldCheckForStarvation())) {
pendingForResourceApps.add(app);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/694e680d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 62430bf..31dd7fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -5079,6 +5079,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
createSchedulingRequestExistingApplication(1024, 1, 1, appAttemptId);
+ scheduler.update();
scheduler.handle(updateEvent);
// no reservation yet
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-6228:
EntityGroupFSTimelineStore should allow configurable cache stores.
Contributed by Li Lu
Posted by su...@apache.org.
YARN-6228: EntityGroupFSTimelineStore should allow configurable cache
stores. Contributed by Li Lu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53d372a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53d372a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53d372a2
Branch: refs/heads/YARN-2915
Commit: 53d372a2550c970f3dd3c49738af3c1789ae589b
Parents: c1a52b0
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 10:58:48 2017 -0800
Committer: Xuan <xg...@apache.org>
Committed: Fri Feb 24 10:59:35 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++++
.../apache/hadoop/yarn/server/timeline/EntityCacheItem.java | 9 +++++++--
3 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d372a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 094a424..cdccec6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1890,6 +1890,9 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX =
TIMELINE_SERVICE_PREFIX + "entity-group-fs-store.";
+ public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE =
+ TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "cache-store-class";
+
public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR =
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "active-dir";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d372a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 53beb5e..368946e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2096,6 +2096,11 @@
</property>
<!-- Timeline Service v1.5 Configuration -->
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.cache-store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore</value>
+ <description>Caching storage timeline server v1.5 is using. </description>
+ </property>
<property>
<name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d372a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
index 7ed7c4a..8df60ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityCacheItem.java
@@ -17,8 +17,10 @@
package org.apache.hadoop.yarn.server.timeline;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -95,8 +97,11 @@ public class EntityCacheItem {
}
if (!appLogs.getDetailLogs().isEmpty()) {
if (store == null) {
- store = new LevelDBCacheTimelineStore(groupId.toString(),
- "LeveldbCache." + groupId);
+ store = ReflectionUtils.newInstance(config.getClass(
+ YarnConfiguration
+ .TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE,
+ MemoryTimelineStore.class, TimelineStore.class),
+ config);
store.init(config);
store.start();
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HADOOP-13321. Deprecate FileSystem
APIs that promote inefficient call patterns. Contributed by Chris Nauroth and
Mingliang Liu
Posted by su...@apache.org.
HADOOP-13321. Deprecate FileSystem APIs that promote inefficient call patterns. Contributed by Chris Nauroth and Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4d4a237
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4d4a237
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4d4a237
Branch: refs/heads/YARN-2915
Commit: a4d4a23785356e6a19d0db3a2dec8ae8cf861273
Parents: a207aa9
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Feb 16 16:25:51 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Feb 23 12:55:40 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/ChecksumFileSystem.java | 2 ++
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 11 +++++++++++
.../java/org/apache/hadoop/fs/ftp/FTPFileSystem.java | 1 +
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 ++
.../hadoop/fs/swift/snative/SwiftNativeFileSystem.java | 2 ++
5 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4d4a237/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index e0ce327..14c1905 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -605,6 +605,7 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
* Rename files/dirs
*/
@Override
+ @SuppressWarnings("deprecation")
public boolean rename(Path src, Path dst) throws IOException {
if (fs.isDirectory(src)) {
return fs.rename(src, dst);
@@ -721,6 +722,7 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
* If src and dst are directories, the copyCrc parameter
* determines whether to copy CRC files.
*/
+ @SuppressWarnings("deprecation")
public void copyToLocalFile(Path src, Path dst, boolean copyCrc)
throws IOException {
if (!fs.isDirectory(src)) { // source is a file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4d4a237/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 55cd97e..ededfa9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1624,6 +1624,11 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/** Check if a path exists.
+ *
+ * It is highly discouraged to call this method back to back with other
+ * {@link #getFileStatus(Path)} calls, as this will involve multiple redundant
+ * RPC calls in HDFS.
+ *
* @param f source path
* @return true if the path exists
* @throws IOException IO failure
@@ -1639,9 +1644,12 @@ public abstract class FileSystem extends Configured implements Closeable {
/** True iff the named path is a directory.
* Note: Avoid using this method. Instead reuse the FileStatus
* returned by getFileStatus() or listStatus() methods.
+ *
* @param f path to check
* @throws IOException IO failure
+ * @deprecated Use {@link #getFileStatus(Path)} instead
*/
+ @Deprecated
public boolean isDirectory(Path f) throws IOException {
try {
return getFileStatus(f).isDirectory();
@@ -1653,9 +1661,12 @@ public abstract class FileSystem extends Configured implements Closeable {
/** True iff the named path is a regular file.
* Note: Avoid using this method. Instead reuse the FileStatus
* returned by {@link #getFileStatus(Path)} or listStatus() methods.
+ *
* @param f path to check
* @throws IOException IO failure
+ * @deprecated Use {@link #getFileStatus(Path)} instead
*/
+ @Deprecated
public boolean isFile(Path f) throws IOException {
try {
return getFileStatus(f).isFile();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4d4a237/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 25fec31..6ce39c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -644,6 +644,7 @@ public class FTPFileSystem extends FileSystem {
* @return
* @throws IOException
*/
+ @SuppressWarnings("deprecation")
private boolean rename(FTPClient client, Path src, Path dst)
throws IOException {
Path workDir = new Path(client.printWorkingDirectory());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4d4a237/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 8b1a6d0..bc47918 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -2142,6 +2142,7 @@ public class S3AFileSystem extends FileSystem {
* {@inheritDoc}
*/
@Override
+ @SuppressWarnings("deprecation")
public boolean isDirectory(Path f) throws IOException {
incrementStatistic(INVOCATION_IS_DIRECTORY);
return super.isDirectory(f);
@@ -2152,6 +2153,7 @@ public class S3AFileSystem extends FileSystem {
* {@inheritDoc}
*/
@Override
+ @SuppressWarnings("deprecation")
public boolean isFile(Path f) throws IOException {
incrementStatistic(INVOCATION_IS_FILE);
return super.isFile(f);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4d4a237/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
index 7f93c38..9217532 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
@@ -201,6 +201,7 @@ public class SwiftNativeFileSystem extends FileSystem {
}
@Override
+ @SuppressWarnings("deprecation")
public boolean isFile(Path f) throws IOException {
try {
FileStatus fileStatus = getFileStatus(f);
@@ -210,6 +211,7 @@ public class SwiftNativeFileSystem extends FileSystem {
}
}
+ @SuppressWarnings("deprecation")
@Override
public boolean isDirectory(Path f) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-3662. Federation Membership
State Store internal APIs.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
new file mode 100644
index 0000000..d4c5451
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterDeregisterRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterDeregisterRequestPBImpl
+ extends SubClusterDeregisterRequest {
+
+ private SubClusterDeregisterRequestProto proto =
+ SubClusterDeregisterRequestProto.getDefaultInstance();
+ private SubClusterDeregisterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterDeregisterRequestPBImpl() {
+ builder = SubClusterDeregisterRequestProto.newBuilder();
+ }
+
+ public SubClusterDeregisterRequestPBImpl(
+ SubClusterDeregisterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterDeregisterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterDeregisterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ SubClusterDeregisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getSubClusterId());
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ return;
+ }
+ builder.setSubClusterId(convertToProtoFormat(subClusterId));
+ }
+
+ @Override
+ public SubClusterState getState() {
+ SubClusterDeregisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getState());
+ }
+
+ @Override
+ public void setState(SubClusterState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearState();
+ return;
+ }
+ builder.setState(convertToProtoFormat(state));
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+ return new SubClusterIdPBImpl(sc);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+ return ((SubClusterIdPBImpl) sc).getProto();
+ }
+
+ private SubClusterState convertFromProtoFormat(SubClusterStateProto state) {
+ return SubClusterState.valueOf(state.name());
+ }
+
+ private SubClusterStateProto convertToProtoFormat(SubClusterState state) {
+ return SubClusterStateProto.valueOf(state.name());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java
new file mode 100644
index 0000000..9e00796
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterDeregisterResponse}.
+ */
+@Private
+@Unstable
+public class SubClusterDeregisterResponsePBImpl
+ extends SubClusterDeregisterResponse {
+
+ private SubClusterDeregisterResponseProto proto =
+ SubClusterDeregisterResponseProto.getDefaultInstance();
+ private SubClusterDeregisterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterDeregisterResponsePBImpl() {
+ builder = SubClusterDeregisterResponseProto.newBuilder();
+ }
+
+ public SubClusterDeregisterResponsePBImpl(
+ SubClusterDeregisterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterDeregisterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
new file mode 100644
index 0000000..ca6b154
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterHeartbeatRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterHeartbeatRequestPBImpl
+ extends SubClusterHeartbeatRequest {
+
+ private SubClusterHeartbeatRequestProto proto =
+ SubClusterHeartbeatRequestProto.getDefaultInstance();
+ private SubClusterHeartbeatRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterId subClusterId = null;
+
+ public SubClusterHeartbeatRequestPBImpl() {
+ builder = SubClusterHeartbeatRequestProto.newBuilder();
+ }
+
+ public SubClusterHeartbeatRequestPBImpl(
+ SubClusterHeartbeatRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterHeartbeatRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterHeartbeatRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterId != null) {
+ builder.setSubClusterId(convertToProtoFormat(this.subClusterId));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterId != null) {
+ return this.subClusterId;
+ }
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ this.subClusterId = convertFromProtoFormat(p.getSubClusterId());
+ return this.subClusterId;
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ }
+ this.subClusterId = subClusterId;
+ }
+
+ @Override
+ public long getLastHeartBeat() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getLastHeartBeat();
+ }
+
+ @Override
+ public void setLastHeartBeat(long time) {
+ maybeInitBuilder();
+ builder.setLastHeartBeat(time);
+ }
+
+ @Override
+ public SubClusterState getState() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getState());
+ }
+
+ @Override
+ public void setState(SubClusterState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearState();
+ return;
+ }
+ builder.setState(convertToProtoFormat(state));
+ }
+
+ @Override
+ public String getCapability() {
+ SubClusterHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasCapability()) ? p.getCapability() : null;
+ }
+
+ @Override
+ public void setCapability(String capability) {
+ maybeInitBuilder();
+ if (capability == null) {
+ builder.clearCapability();
+ return;
+ }
+ builder.setCapability(capability);
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto clusterId) {
+ return new SubClusterIdPBImpl(clusterId);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId clusterId) {
+ return ((SubClusterIdPBImpl) clusterId).getProto();
+ }
+
+ private SubClusterState convertFromProtoFormat(SubClusterStateProto state) {
+ return SubClusterState.valueOf(state.name());
+ }
+
+ private SubClusterStateProto convertToProtoFormat(SubClusterState state) {
+ return SubClusterStateProto.valueOf(state.name());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java
new file mode 100644
index 0000000..2020c1a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterHeartbeatResponse}.
+ */
+@Private
+@Unstable
+public class SubClusterHeartbeatResponsePBImpl
+ extends SubClusterHeartbeatResponse {
+
+ private SubClusterHeartbeatResponseProto proto =
+ SubClusterHeartbeatResponseProto.getDefaultInstance();
+ private SubClusterHeartbeatResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterHeartbeatResponsePBImpl() {
+ builder = SubClusterHeartbeatResponseProto.newBuilder();
+ }
+
+ public SubClusterHeartbeatResponsePBImpl(
+ SubClusterHeartbeatResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterHeartbeatResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java
new file mode 100644
index 0000000..1bf96bf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterIdPBImpl.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterId}.
+ */
+@Private
+@Unstable
+public class SubClusterIdPBImpl extends SubClusterId {
+
+ private SubClusterIdProto proto = SubClusterIdProto.getDefaultInstance();
+ private SubClusterIdProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterIdPBImpl() {
+ builder = SubClusterIdProto.newBuilder();
+ }
+
+ public SubClusterIdPBImpl(SubClusterIdProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterIdProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterIdProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public String getId() {
+ SubClusterIdProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getId();
+ }
+
+ @Override
+ protected void setId(String subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearId();
+ return;
+ }
+ builder.setId(subClusterId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
new file mode 100644
index 0000000..b650b5f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterInfo}.
+ */
+@Private
+@Unstable
+public class SubClusterInfoPBImpl extends SubClusterInfo {
+
+ private SubClusterInfoProto proto = SubClusterInfoProto.getDefaultInstance();
+ private SubClusterInfoProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterId subClusterId = null;
+
+ public SubClusterInfoPBImpl() {
+ builder = SubClusterInfoProto.newBuilder();
+ }
+
+ public SubClusterInfoPBImpl(SubClusterInfoProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterInfoProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterInfoProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterId != null) {
+ builder.setSubClusterId(convertToProtoFormat(this.subClusterId));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterId != null) {
+ return this.subClusterId;
+ }
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ this.subClusterId = convertFromProtoFormat(p.getSubClusterId());
+ return this.subClusterId;
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ }
+ this.subClusterId = subClusterId;
+ }
+
+ @Override
+ public String getAMRMServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasAMRMServiceAddress()) ? p.getAMRMServiceAddress() : null;
+ }
+
+ @Override
+ public void setAMRMServiceAddress(String amRMServiceAddress) {
+ maybeInitBuilder();
+ if (amRMServiceAddress == null) {
+ builder.clearAMRMServiceAddress();
+ return;
+ }
+ builder.setAMRMServiceAddress(amRMServiceAddress);
+ }
+
+ @Override
+ public String getClientRMServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasClientRMServiceAddress()) ? p.getClientRMServiceAddress()
+ : null;
+ }
+
+ @Override
+ public void setClientRMServiceAddress(String clientRMServiceAddress) {
+ maybeInitBuilder();
+ if (clientRMServiceAddress == null) {
+ builder.clearClientRMServiceAddress();
+ return;
+ }
+ builder.setClientRMServiceAddress(clientRMServiceAddress);
+ }
+
+ @Override
+ public String getRMAdminServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasRMAdminServiceAddress()) ? p.getRMAdminServiceAddress() : null;
+ }
+
+ @Override
+ public void setRMAdminServiceAddress(String rmAdminServiceAddress) {
+ maybeInitBuilder();
+ if (rmAdminServiceAddress == null) {
+ builder.clearRMAdminServiceAddress();
+ return;
+ }
+ builder.setRMAdminServiceAddress(rmAdminServiceAddress);
+ }
+
+ @Override
+ public String getRMWebServiceAddress() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasRMWebServiceAddress()) ? p.getRMWebServiceAddress() : null;
+ }
+
+ @Override
+ public void setRMWebServiceAddress(String rmWebServiceAddress) {
+ maybeInitBuilder();
+ if (rmWebServiceAddress == null) {
+ builder.clearRMWebServiceAddress();
+ return;
+ }
+ builder.setRMWebServiceAddress(rmWebServiceAddress);
+ }
+
+ @Override
+ public long getLastHeartBeat() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getLastHeartBeat();
+ }
+
+ @Override
+ public void setLastHeartBeat(long time) {
+ maybeInitBuilder();
+ builder.setLastHeartBeat(time);
+ }
+
+ @Override
+ public SubClusterState getState() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasState()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getState());
+ }
+
+ @Override
+ public void setState(SubClusterState state) {
+ maybeInitBuilder();
+ if (state == null) {
+ builder.clearState();
+ return;
+ }
+ builder.setState(convertToProtoFormat(state));
+ }
+
+ @Override
+ public long getLastStartTime() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasLastStartTime()) ? p.getLastStartTime() : 0;
+ }
+
+ @Override
+ public void setLastStartTime(long lastStartTime) {
+ Preconditions.checkNotNull(builder);
+ builder.setLastStartTime(lastStartTime);
+ }
+
+ @Override
+ public String getCapability() {
+ SubClusterInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasCapability()) ? p.getCapability() : null;
+ }
+
+ @Override
+ public void setCapability(String capability) {
+ maybeInitBuilder();
+ if (capability == null) {
+ builder.clearCapability();
+ return;
+ }
+ builder.setCapability(capability);
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto clusterId) {
+ return new SubClusterIdPBImpl(clusterId);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId clusterId) {
+ return ((SubClusterIdPBImpl) clusterId).getProto();
+ }
+
+ private SubClusterState convertFromProtoFormat(SubClusterStateProto state) {
+ return SubClusterState.valueOf(state.name());
+ }
+
+ private SubClusterStateProto convertToProtoFormat(SubClusterState state) {
+ return SubClusterStateProto.valueOf(state.name());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
new file mode 100644
index 0000000..3429cc9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterRegisterRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterRegisterRequestPBImpl extends SubClusterRegisterRequest {
+
+ private SubClusterRegisterRequestProto proto =
+ SubClusterRegisterRequestProto.getDefaultInstance();
+ private SubClusterRegisterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterInfo subClusterInfo = null;
+
+ public SubClusterRegisterRequestPBImpl() {
+ builder = SubClusterRegisterRequestProto.newBuilder();
+ }
+
+ public SubClusterRegisterRequestPBImpl(SubClusterRegisterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterRegisterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterRegisterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterInfo != null) {
+ builder.setSubClusterInfo(convertToProtoFormat(this.subClusterInfo));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterInfo getSubClusterInfo() {
+ SubClusterRegisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterInfo != null) {
+ return this.subClusterInfo;
+ }
+ if (!p.hasSubClusterInfo()) {
+ return null;
+ }
+ this.subClusterInfo = convertFromProtoFormat(p.getSubClusterInfo());
+ return this.subClusterInfo;
+ }
+
+ @Override
+ public void setSubClusterInfo(SubClusterInfo subClusterInfo) {
+ maybeInitBuilder();
+ if (subClusterInfo == null) {
+ builder.clearSubClusterInfo();
+ }
+ this.subClusterInfo = subClusterInfo;
+ }
+
+ private SubClusterInfo convertFromProtoFormat(
+ SubClusterInfoProto clusterInfo) {
+ return new SubClusterInfoPBImpl(clusterInfo);
+ }
+
+ private SubClusterInfoProto convertToProtoFormat(SubClusterInfo clusterInfo) {
+ return ((SubClusterInfoPBImpl) clusterInfo).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java
new file mode 100644
index 0000000..68930e3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterRegisterResponse}.
+ */
+@Private
+@Unstable
+public class SubClusterRegisterResponsePBImpl
+ extends SubClusterRegisterResponse {
+
+ private SubClusterRegisterResponseProto proto =
+ SubClusterRegisterResponseProto.getDefaultInstance();
+ private SubClusterRegisterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterRegisterResponsePBImpl() {
+ builder = SubClusterRegisterResponseProto.newBuilder();
+ }
+
+ public SubClusterRegisterResponsePBImpl(
+ SubClusterRegisterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterRegisterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java
new file mode 100644
index 0000000..2f85c48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java
new file mode 100644
index 0000000..9a9b282
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
new file mode 100644
index 0000000..1b2e53e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.federation.proto";
+option java_outer_classname = "YarnServerFederationProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+import "yarn_protos.proto";
+import "yarn_server_common_protos.proto";
+
+message SubClusterIdProto {
+ optional string id = 1;
+}
+
+enum SubClusterStateProto {
+ SC_NEW = 1;
+ SC_RUNNING = 2;
+ SC_UNHEALTHY = 3;
+ SC_DECOMMISSIONING = 4;
+ SC_LOST = 5;
+ SC_UNREGISTERED = 6;
+ SC_DECOMMISSIONED = 7;
+}
+
+message SubClusterInfoProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+ optional string aMRM_service_address = 2;
+ optional string client_rM_service_address = 3;
+ optional string rM_admin_service_address = 4;
+ optional string rM_web_service_address = 5;
+ optional int64 lastHeartBeat = 6;
+ optional SubClusterStateProto state = 7;
+ optional int64 lastStartTime = 8;
+ optional string capability = 9;
+}
+
+message SubClusterRegisterRequestProto {
+ optional SubClusterInfoProto sub_cluster_info = 1;
+}
+
+message SubClusterRegisterResponseProto {
+}
+
+message SubClusterHeartbeatRequestProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+ optional int64 lastHeartBeat = 2;
+ optional SubClusterStateProto state = 3;
+ optional string capability = 4;
+}
+
+message SubClusterHeartbeatResponseProto {
+}
+
+message SubClusterDeregisterRequestProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+ optional SubClusterStateProto state = 2;
+}
+
+message SubClusterDeregisterResponseProto {
+}
+
+message GetSubClusterInfoRequestProto {
+ optional SubClusterIdProto sub_cluster_id = 1;
+}
+
+message GetSubClusterInfoResponseProto {
+ optional SubClusterInfoProto sub_cluster_info = 1;
+}
+
+message GetSubClustersInfoRequestProto {
+ optional bool filter_inactive_subclusters = 1 [default = true];
+}
+
+message GetSubClustersInfoResponseProto {
+ repeated SubClusterInfoProto sub_cluster_infos = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
new file mode 100644
index 0000000..681edb1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.yarn.api.BasePBImplRecordsTest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterHeartbeatRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterHeartbeatResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test class for federation protocol records.
+ */
+public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ generateByNewInstance(ApplicationId.class);
+ generateByNewInstance(Version.class);
+ generateByNewInstance(SubClusterId.class);
+ generateByNewInstance(SubClusterInfo.class);
+ }
+
+ @Test
+ public void testSubClusterId() throws Exception {
+ validatePBImplRecord(SubClusterIdPBImpl.class, SubClusterIdProto.class);
+ }
+
+ @Test
+ public void testSubClusterInfo() throws Exception {
+ validatePBImplRecord(SubClusterInfoPBImpl.class, SubClusterInfoProto.class);
+ }
+
+ @Test
+ public void testSubClusterRegisterRequest() throws Exception {
+ validatePBImplRecord(SubClusterRegisterRequestPBImpl.class,
+ SubClusterRegisterRequestProto.class);
+ }
+
+ @Test
+ public void testSubClusterRegisterResponse() throws Exception {
+ validatePBImplRecord(SubClusterRegisterResponsePBImpl.class,
+ SubClusterRegisterResponseProto.class);
+ }
+
+ @Test
+ public void testSubClusterDeregisterRequest() throws Exception {
+ validatePBImplRecord(SubClusterDeregisterRequestPBImpl.class,
+ SubClusterDeregisterRequestProto.class);
+ }
+
+ @Test
+ public void testSubClusterDeregisterResponse() throws Exception {
+ validatePBImplRecord(SubClusterDeregisterResponsePBImpl.class,
+ SubClusterDeregisterResponseProto.class);
+ }
+
+ @Test
+ public void testSubClusterHeartbeatRequest() throws Exception {
+ validatePBImplRecord(SubClusterHeartbeatRequestPBImpl.class,
+ SubClusterHeartbeatRequestProto.class);
+ }
+
+ @Test
+ public void testSubClusterHeartbeatResponse() throws Exception {
+ validatePBImplRecord(SubClusterHeartbeatResponsePBImpl.class,
+ SubClusterHeartbeatResponseProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterRequest() throws Exception {
+ validatePBImplRecord(GetSubClusterInfoRequestPBImpl.class,
+ GetSubClusterInfoRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterResponse() throws Exception {
+ validatePBImplRecord(GetSubClusterInfoResponsePBImpl.class,
+ GetSubClusterInfoResponseProto.class);
+ }
+
+ @Test
+ public void testGetSubClustersInfoRequest() throws Exception {
+ validatePBImplRecord(GetSubClustersInfoRequestPBImpl.class,
+ GetSubClustersInfoRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClustersInfoResponse() throws Exception {
+ validatePBImplRecord(GetSubClustersInfoResponsePBImpl.class,
+ GetSubClustersInfoResponseProto.class);
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-5905. Update the RM webapp
host that is reported as part of Federation membership to current primary
RM's IP.
Posted by su...@apache.org.
YARN-5905. Update the RM webapp host that is reported as part of Federation membership to current primary RM's IP.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb7a7a7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb7a7a7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb7a7a7a
Branch: refs/heads/YARN-2915
Commit: eb7a7a7ab622ca2bf4681b3f36021846417feab1
Parents: d028a9b
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 22 18:30:40 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../federation/FederationStateStoreService.java | 4 ++--
.../federation/TestFederationRMStateStoreService.java | 11 ++++++++++-
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb7a7a7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
index 9a01d7e..530184f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
@@ -177,8 +177,8 @@ public class FederationStateStoreService extends AbstractService
config.getSocketAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
- String webAppAddress =
- WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(config);
+ String webAppAddress = getServiceAddress(NetUtils
+ .createSocketAddr(WebAppUtils.getRMWebAppURLWithScheme(config)));
SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
amRMAddress, clientRMAddress, rmAdminAddress, webAppAddress,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb7a7a7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
index 30f69b5..d92a793 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.federation;
import java.io.IOException;
import java.io.StringReader;
+import java.net.UnknownHostException;
import javax.xml.bind.JAXBException;
@@ -157,12 +158,20 @@ public class TestFederationRMStateStoreService {
}
private String checkSubClusterInfo(SubClusterState state)
- throws YarnException {
+ throws YarnException, UnknownHostException {
Assert.assertNotNull(stateStore.getSubCluster(request));
SubClusterInfo response =
stateStore.getSubCluster(request).getSubClusterInfo();
Assert.assertEquals(state, response.getState());
Assert.assertTrue(response.getLastHeartBeat() >= lastHearbeatTS);
+ String expectedAddress =
+ (response.getClientRMServiceAddress().split(":"))[0];
+ Assert.assertEquals(expectedAddress,
+ (response.getAMRMServiceAddress().split(":"))[0]);
+ Assert.assertEquals(expectedAddress,
+ (response.getRMAdminServiceAddress().split(":"))[0]);
+ Assert.assertEquals(expectedAddress,
+ (response.getRMWebServiceAddress().split(":"))[0]);
lastHearbeatTS = response.getLastHeartBeat();
return response.getCapability();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-5325. Stateless ARMRMProxy
policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index e57709f..5de749f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -17,8 +17,8 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
@@ -30,34 +30,27 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
-import java.util.Map;
-
/**
* This implements a simple load-balancing policy. The policy "weights" are
* binary 0/1 values that enable/disable each sub-cluster, and the policy peaks
* the sub-cluster with the least load to forward this application.
*/
-public class LoadBasedRouterPolicy
- extends BaseWeightedRouterPolicy {
-
- private static final Log LOG =
- LogFactory.getLog(LoadBasedRouterPolicy.class);
+public class LoadBasedRouterPolicy extends AbstractRouterPolicy {
@Override
- public void reinitialize(FederationPolicyInitializationContext
- federationPolicyContext)
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
// remember old policyInfo
WeightedPolicyInfo tempPolicy = getPolicyInfo();
- //attempt new initialization
- super.reinitialize(federationPolicyContext);
+ // attempt new initialization
+ super.reinitialize(policyContext);
- //check extra constraints
+ // check extra constraints
for (Float weight : getPolicyInfo().getRouterPolicyWeights().values()) {
if (weight != 0 && weight != 1) {
- //reset to old policyInfo if check fails
+ // reset to old policyInfo if check fails
setPolicyInfo(tempPolicy);
throw new FederationPolicyInitializationException(
this.getClass().getCanonicalName()
@@ -69,18 +62,16 @@ public class LoadBasedRouterPolicy
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
- Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
- .getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> weights =
+ getPolicyInfo().getRouterPolicyWeights();
SubClusterIdInfo chosen = null;
long currBestMem = -1;
- for (Map.Entry<SubClusterId, SubClusterInfo> entry :
- activeSubclusters
+ for (Map.Entry<SubClusterId, SubClusterInfo> entry : activeSubclusters
.entrySet()) {
SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey());
if (weights.containsKey(id) && weights.get(id) > 0) {
@@ -95,8 +86,7 @@ public class LoadBasedRouterPolicy
return chosen.toId();
}
- private long getAvailableMemory(SubClusterInfo value)
- throws YarnException {
+ private long getAvailableMemory(SubClusterInfo value) throws YarnException {
try {
long mem = -1;
JSONObject obj = new JSONObject(value.getCapability());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
index a8ac5f7..bc3a1f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -17,39 +17,32 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import java.util.Map;
-
/**
* This implements a policy that interprets "weights" as a ordered list of
* preferences among sub-clusters. Highest weight among active subclusters is
* chosen.
*/
-public class PriorityRouterPolicy
- extends BaseWeightedRouterPolicy {
-
- private static final Log LOG =
- LogFactory.getLog(PriorityRouterPolicy.class);
+public class PriorityRouterPolicy extends AbstractRouterPolicy {
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
// This finds the sub-cluster with the highest weight among the
// currently active ones.
- Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
- .getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> weights =
+ getPolicyInfo().getRouterPolicyWeights();
SubClusterId chosen = null;
Float currentBest = Float.MIN_VALUE;
for (SubClusterId id : activeSubclusters.keySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
index 1774961..b8f9cc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -17,6 +17,11 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
@@ -25,11 +30,6 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPo
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
/**
* This simple policy picks at uniform random among any of the currently active
* subclusters. This policy is easy to use and good for testing.
@@ -39,7 +39,7 @@ import java.util.Random;
* of the "weights", in which case the {@link UniformRandomRouterPolicy} send
* load to them, while {@code WeightedRandomRouterPolicy} does not.
*/
-public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
+public class UniformRandomRouterPolicy extends AbstractRouterPolicy {
private Random rand;
@@ -49,14 +49,14 @@ public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
@Override
public void reinitialize(
- FederationPolicyInitializationContext federationPolicyContext)
+ FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
FederationPolicyInitializationContextValidator
- .validate(federationPolicyContext, this.getClass().getCanonicalName());
+ .validate(policyContext, this.getClass().getCanonicalName());
- //note: this overrides BaseWeighterRouterPolicy and ignores the weights
+ // note: this overrides AbstractRouterPolicy and ignores the weights
- setPolicyContext(federationPolicyContext);
+ setPolicyContext(policyContext);
}
/**
@@ -64,21 +64,19 @@ public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
* depend on the weights in the policy).
*
* @param appSubmissionContext the context for the app being submitted
- * (ignored).
+ * (ignored).
*
* @return a randomly chosen subcluster.
*
* @throws YarnException if there are no active subclusters.
*/
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
- List<SubClusterId> list =
- new ArrayList<>(activeSubclusters.keySet());
+ List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
return list.get(rand.nextInt(list.size()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
index 0777677..ac75ae9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -18,32 +18,30 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+import java.util.Random;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-
-import java.util.Map;
-import java.util.Random;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This policy implements a weighted random sample among currently active
* sub-clusters.
*/
-public class WeightedRandomRouterPolicy
- extends BaseWeightedRouterPolicy {
+public class WeightedRandomRouterPolicy extends AbstractRouterPolicy {
- private static final Log LOG =
- LogFactory.getLog(WeightedRandomRouterPolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WeightedRandomRouterPolicy.class);
private Random rand = new Random(System.currentTimeMillis());
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
@@ -52,13 +50,13 @@ public class WeightedRandomRouterPolicy
// changes dynamically (and this would unfairly spread the load to
// sub-clusters adjacent to an inactive one), hence we need to count/scan
// the list and based on weight pick the next sub-cluster.
- Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
- .getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> weights =
+ getPolicyInfo().getRouterPolicyWeights();
float totActiveWeight = 0;
- for(Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()){
- if(entry.getKey()!=null && activeSubclusters.containsKey(entry.getKey()
- .toId())){
+ for (Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()) {
+ if (entry.getKey() != null
+ && activeSubclusters.containsKey(entry.getKey().toId())) {
totActiveWeight += entry.getValue();
}
}
@@ -73,7 +71,7 @@ public class WeightedRandomRouterPolicy
return id;
}
}
- //should never happen
+ // should never happen
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
index 5d0fcb6..e445ac3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
@@ -17,4 +17,3 @@
*/
/** Router policies. **/
package org.apache.hadoop.yarn.server.federation.policies.router;
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
index 8238633..6b4f60c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
@@ -57,11 +57,11 @@ public abstract class AbstractSubClusterResolver implements SubClusterResolver {
return rackToSubClusters.get(rackname);
}
- protected Map<String, SubClusterId> getNodeToSubCluster() {
+ public Map<String, SubClusterId> getNodeToSubCluster() {
return nodeToSubCluster;
}
- protected Map<String, Set<SubClusterId>> getRackToSubClusters() {
+ public Map<String, Set<SubClusterId>> getRackToSubClusters() {
return rackToSubClusters;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
index 8da92b9..ba897da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -22,14 +22,17 @@ import static org.mockito.Mockito.mock;
import java.nio.ByteBuffer;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -49,6 +52,7 @@ public abstract class BaseFederationPoliciesTest {
private ApplicationSubmissionContext applicationSubmissionContext =
mock(ApplicationSubmissionContext.class);
private Random rand = new Random();
+ private SubClusterId homeSubCluster;
@Test
public void testReinitilialize() throws YarnException {
@@ -88,16 +92,22 @@ public abstract class BaseFederationPoliciesTest {
getPolicy().reinitialize(fpc);
}
- @Test(expected = NoActiveSubclustersException.class)
+ @Test(expected = FederationPolicyException.class)
public void testNoSubclusters() throws YarnException {
// empty the activeSubclusters map
FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
getPolicyInfo(), new HashMap<>());
- ConfigurableFederationPolicy currentPolicy = getPolicy();
- if (currentPolicy instanceof FederationRouterPolicy) {
- ((FederationRouterPolicy) currentPolicy)
+ ConfigurableFederationPolicy localPolicy = getPolicy();
+ if (localPolicy instanceof FederationRouterPolicy) {
+ ((FederationRouterPolicy) localPolicy)
.getHomeSubcluster(getApplicationSubmissionContext());
+ } else {
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+ ((FederationAMRMProxyPolicy) localPolicy)
+ .splitResourceRequests(resourceRequests);
}
}
@@ -152,4 +162,12 @@ public abstract class BaseFederationPoliciesTest {
this.rand = rand;
}
+ public SubClusterId getHomeSubCluster() {
+ return homeSubCluster;
+ }
+
+ public void setHomeSubCluster(SubClusterId homeSubCluster) {
+ this.homeSubCluster = homeSubCluster;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index e840b3f..c79fd2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -16,22 +16,20 @@
* limitations under the License.
*/
-
package org.apache.hadoop.yarn.server.federation.policies;
+import java.nio.ByteBuffer;
+
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.junit.Before;
import org.junit.Test;
-import java.nio.ByteBuffer;
-
/**
* Test class for {@link FederationPolicyInitializationContextValidator}.
*/
@@ -45,11 +43,10 @@ public class TestFederationPolicyInitializationContextValidator {
@Before
public void setUp() throws Exception {
goodFacade = FederationPoliciesTestUtil.initFacade();
- goodConfig =
- new MockPolicyManager().serializeConf();
- goodSR =FederationPoliciesTestUtil.initResolver();
- context = new
- FederationPolicyInitializationContext(goodConfig, goodSR, goodFacade);
+ goodConfig = new MockPolicyManager().serializeConf();
+ goodSR = FederationPoliciesTestUtil.initResolver();
+ context = new FederationPolicyInitializationContext(goodConfig, goodSR,
+ goodFacade);
}
@Test
@@ -100,8 +97,7 @@ public class TestFederationPolicyInitializationContextValidator {
@Override
public FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationAMRMProxyPolicy oldInstance)
throws FederationPolicyInitializationException {
return null;
@@ -109,8 +105,7 @@ public class TestFederationPolicyInitializationContextValidator {
@Override
public FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
+ FederationPolicyInitializationContext policyContext,
FederationRouterPolicy oldInstance)
throws FederationPolicyInitializationException {
return null;
@@ -120,8 +115,8 @@ public class TestFederationPolicyInitializationContextValidator {
public SubClusterPolicyConfiguration serializeConf()
throws FederationPolicyInitializationException {
ByteBuffer buf = ByteBuffer.allocate(0);
- return SubClusterPolicyConfiguration
- .newInstance("queue1", this.getClass().getCanonicalName(), buf);
+ return SubClusterPolicyConfiguration.newInstance("queue1",
+ this.getClass().getCanonicalName(), buf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
new file mode 100644
index 0000000..a21f53d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link BroadcastAMRMProxyPolicy}.
+ */
+public class TestBroadcastAMRMProxyFederationPolicy
+ extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new BroadcastAMRMProxyPolicy());
+ // needed for base test to work
+ setPolicyInfo(mock(WeightedPolicyInfo.class));
+
+ for (int i = 1; i <= 2; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ mock(WeightedPolicyInfo.class), getActiveSubclusters());
+
+ }
+
+ @Test
+ public void testSplitAllocateRequest() throws Exception {
+ // verify the request is broadcasted to all subclusters
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+ Assert.assertTrue(response.size() == 2);
+ for (Map.Entry<SubClusterId, List<ResourceRequest>> entry : response
+ .entrySet()) {
+ Assert.assertTrue(getActiveSubclusters().get(entry.getKey()) != null);
+ for (ResourceRequest r : entry.getValue()) {
+ Assert.assertTrue(resourceRequests.contains(r));
+ }
+ }
+ for (SubClusterId subClusterId : getActiveSubclusters().keySet()) {
+ for (ResourceRequest r : response.get(subClusterId)) {
+ Assert.assertTrue(resourceRequests.contains(r));
+ }
+ }
+ }
+
+ @Test
+ public void testNotifyOfResponse() throws Exception {
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ try {
+ ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
+ SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));
+ Assert.fail();
+ } catch (FederationPolicyException f) {
+ System.out.println("Expected: " + f.getMessage());
+ }
+
+ ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
+ SubClusterId.newInstance("sc1"), mock(AllocateResponse.class));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
new file mode 100644
index 0000000..2654a06
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple test class for the {@link LocalityMulticastAMRMProxyPolicy}.
+ */
+public class TestLocalityMulticastAMRMProxyPolicy
+ extends BaseFederationPoliciesTest {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestLocalityMulticastAMRMProxyPolicy.class);
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new LocalityMulticastAMRMProxyPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 subclusters with a 5% chance of being inactive
+ for (int i = 0; i < 6; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("subcluster" + i);
+ // sub-cluster 3 is not active
+ if (i != 3) {
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ float weight = 1 / 10f;
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ // sub-cluster 4 is "disabled" in the weights
+ if (i == 4) {
+ routerWeights.put(sc, 0f);
+ amrmWeights.put(sc, 0f);
+ }
+ }
+
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+ getPolicyInfo().setHeadroomAlpha(0.5f);
+ setHomeSubCluster(SubClusterId.newInstance("homesubcluster"));
+
+ }
+
+ @Test
+ public void testReinitilialize() throws YarnException {
+ initializePolicy();
+ }
+
+ private void initializePolicy() throws YarnException {
+ setFederationPolicyContext(new FederationPolicyInitializationContext());
+ SubClusterResolver resolver = FederationPoliciesTestUtil.initResolver();
+ getFederationPolicyContext().setFederationSubclusterResolver(resolver);
+ ByteBuffer buf = getPolicyInfo().toByteBuffer();
+ getFederationPolicyContext().setSubClusterPolicyConfiguration(
+ SubClusterPolicyConfiguration.newInstance("queue1",
+ getPolicy().getClass().getCanonicalName(), buf));
+ getFederationPolicyContext().setHomeSubcluster(getHomeSubCluster());
+ FederationPoliciesTestUtil.initializePolicyContext(
+ getFederationPolicyContext(), getPolicy(), getPolicyInfo(),
+ getActiveSubclusters());
+ }
+
+ @Test
+ public void testSplitBasedOnHeadroom() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+ // Configure policy to be 100% headroom based
+ getPolicyInfo().setHeadroomAlpha(1.0f);
+
+ initializePolicy();
+ List<ResourceRequest> resourceRequests = createSimpleRequest();
+
+ prepPolicyWithHeadroom();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ // pretty print requests
+ LOG.info("Initial headroom");
+ prettyPrintRequests(response);
+
+ validateSplit(response, resourceRequests);
+
+ // based on headroom, we expect 75 containers to got to subcluster0,
+ // as it advertise lots of headroom (100), no containers for sublcuster1
+ // as it advertise zero headroom, 1 to subcluster 2 (as it advertise little
+ // headroom (1), and 25 to subcluster5 which has unknown headroom, and so
+ // it gets 1/4th of the load
+ checkExpectedAllocation(response, "subcluster0", 1, 75);
+ checkExpectedAllocation(response, "subcluster1", 1, -1);
+ checkExpectedAllocation(response, "subcluster2", 1, 1);
+ checkExpectedAllocation(response, "subcluster5", 1, 25);
+
+ // notify a change in headroom and try again
+ AllocateResponse ar = getAllocateResponseWithTargetHeadroom(100);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster2"), ar);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ LOG.info("After headroom update");
+ prettyPrintRequests(response);
+ validateSplit(response, resourceRequests);
+
+ // we simulated a change in headroom for subcluster2, which will now
+ // have the same headroom of subcluster0 and so it splits the requests
+ // note that the total is still less or equal to (userAsk + numSubClusters)
+ checkExpectedAllocation(response, "subcluster0", 1, 38);
+ checkExpectedAllocation(response, "subcluster1", 1, -1);
+ checkExpectedAllocation(response, "subcluster2", 1, 38);
+ checkExpectedAllocation(response, "subcluster5", 1, 25);
+
+ }
+
+ @Test(timeout = 5000)
+ public void testStressPolicy() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+ // Configure policy to be 100% headroom based
+ getPolicyInfo().setHeadroomAlpha(1.0f);
+
+ initializePolicy();
+
+ int numRR = 1000;
+ List<ResourceRequest> resourceRequests = createLargeRandomList(numRR);
+
+ prepPolicyWithHeadroom();
+
+ int numIterations = 1000;
+ long tstart = System.currentTimeMillis();
+ for (int i = 0; i < numIterations; i++) {
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+ validateSplit(response, resourceRequests);
+ }
+ long tend = System.currentTimeMillis();
+
+ LOG.info("Performed " + numIterations + " policy invocations (and "
+ + "validations) in " + (tend - tstart) + "ms");
+ }
+
+ @Test
+ public void testFWDAllZeroANY() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+ // Configure policy to be 100% headroom based
+ getPolicyInfo().setHeadroomAlpha(0.5f);
+
+ initializePolicy();
+ List<ResourceRequest> resourceRequests = createZeroSizedANYRequest();
+
+ // this receives responses from sc0,sc1,sc2
+ prepPolicyWithHeadroom();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ // we expect all three to appear for a zero-sized ANY
+
+ // pretty print requests
+ prettyPrintRequests(response);
+
+ validateSplit(response, resourceRequests);
+
+ // we expect the zero size request to be sent to the first 3 rm (due to
+ // the fact that we received responses only from these 3 sublcusters)
+ checkExpectedAllocation(response, "subcluster0", 1, 0);
+ checkExpectedAllocation(response, "subcluster1", 1, 0);
+ checkExpectedAllocation(response, "subcluster2", 1, 0);
+ checkExpectedAllocation(response, "subcluster3", -1, -1);
+ checkExpectedAllocation(response, "subcluster4", -1, -1);
+ checkExpectedAllocation(response, "subcluster5", -1, -1);
+ }
+
+ @Test
+ public void testSplitBasedOnHeadroomAndWeights() throws Exception {
+
+ // Tests how the headroom info are used to split based on the capacity
+ // each RM claims to give us.
+
+ // Configure policy to be 50% headroom based and 50% weight based
+ getPolicyInfo().setHeadroomAlpha(0.5f);
+
+ initializePolicy();
+ List<ResourceRequest> resourceRequests = createSimpleRequest();
+
+ prepPolicyWithHeadroom();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ // pretty print requests
+ prettyPrintRequests(response);
+
+ validateSplit(response, resourceRequests);
+
+ // in this case the headroom allocates 50 containers, while weights allocate
+ // the rest. due to weights we have 12.5 (round to 13) containers for each
+ // sublcuster, the rest is due to headroom.
+ checkExpectedAllocation(response, "subcluster0", 1, 50);
+ checkExpectedAllocation(response, "subcluster1", 1, 13);
+ checkExpectedAllocation(response, "subcluster2", 1, 13);
+ checkExpectedAllocation(response, "subcluster3", -1, -1);
+ checkExpectedAllocation(response, "subcluster4", -1, -1);
+ checkExpectedAllocation(response, "subcluster5", 1, 25);
+
+ }
+
+ private void prepPolicyWithHeadroom() throws YarnException {
+ AllocateResponse ar = getAllocateResponseWithTargetHeadroom(100);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster0"), ar);
+
+ ar = getAllocateResponseWithTargetHeadroom(0);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster1"), ar);
+
+ ar = getAllocateResponseWithTargetHeadroom(1);
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .notifyOfResponse(SubClusterId.newInstance("subcluster2"), ar);
+ }
+
+ private AllocateResponse getAllocateResponseWithTargetHeadroom(
+ int numContainers) {
+ return AllocateResponse.newInstance(0, null, null,
+ Collections.<NodeReport> emptyList(),
+ Resource.newInstance(numContainers * 1024, numContainers), null, 10,
+ null, Collections.<NMToken> emptyList());
+ }
+
+ @Test
+ public void testSplitAllocateRequest() throws Exception {
+
+ // Test a complex List<ResourceRequest> is split correctly
+ initializePolicy();
+
+ // modify default initialization to include a "homesubcluster"
+ // which we will use as the default for when nodes or racks are unknown
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(getHomeSubCluster());
+ getActiveSubclusters().put(getHomeSubCluster(), sci);
+ SubClusterIdInfo sc = new SubClusterIdInfo(getHomeSubCluster().getId());
+
+ getPolicyInfo().getRouterPolicyWeights().put(sc, 0.1f);
+ getPolicyInfo().getAMRMPolicyWeights().put(sc, 0.1f);
+
+ FederationPoliciesTestUtil.initializePolicyContext(
+ getFederationPolicyContext(), getPolicy(), getPolicyInfo(),
+ getActiveSubclusters());
+
+ List<ResourceRequest> resourceRequests = createComplexRequest();
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+
+ validateSplit(response, resourceRequests);
+ prettyPrintRequests(response);
+
+ // we expect 4 entry for home subcluster (3 for request-id 4, and a part
+ // of the broadcast of request-id 2
+ checkExpectedAllocation(response, getHomeSubCluster().getId(), 4, 23);
+
+ // for subcluster0 we expect 3 entry from request-id 0, and 3 from
+ // request-id 3, as well as part of the request-id 2 broadast
+ checkExpectedAllocation(response, "subcluster0", 7, 26);
+
+ // we expect 5 entry for subcluster1 (4 from request-id 1, and part
+ // of the broadcast of request-id 2
+ checkExpectedAllocation(response, "subcluster1", 5, 25);
+
+ // sub-cluster 2 should contain 3 entry from request-id 1 and 1 from the
+ // broadcast of request-id 2, and no request-id 0
+ checkExpectedAllocation(response, "subcluster2", 4, 23);
+
+ // subcluster id 3, 4 should not appear (due to weights or active/inactive)
+ checkExpectedAllocation(response, "subcluster3", -1, -1);
+ checkExpectedAllocation(response, "subcluster4", -1, -1);
+
+ // subcluster5 should get only part of the request-id 2 broadcast
+ checkExpectedAllocation(response, "subcluster5", 1, 20);
+
+ // check that the allocations that show up are what expected
+ for (ResourceRequest rr : response.get(getHomeSubCluster())) {
+ Assert.assertTrue(rr.getAllocationRequestId() == 4L
+ || rr.getAllocationRequestId() == 2L);
+ }
+
+ for (ResourceRequest rr : response.get(getHomeSubCluster())) {
+ Assert.assertTrue(rr.getAllocationRequestId() != 1L);
+ }
+
+ List<ResourceRequest> rrs =
+ response.get(SubClusterId.newInstance("subcluster0"));
+ for (ResourceRequest rr : rrs) {
+ Assert.assertTrue(rr.getAllocationRequestId() != 1L);
+ }
+
+ for (ResourceRequest rr : response
+ .get(SubClusterId.newInstance("subcluster2"))) {
+ Assert.assertTrue(rr.getAllocationRequestId() != 0L);
+ }
+
+ for (ResourceRequest rr : response
+ .get(SubClusterId.newInstance("subcluster5"))) {
+ Assert.assertTrue(rr.getAllocationRequestId() >= 2);
+ Assert.assertTrue(rr.getRelaxLocality());
+ }
+ }
+
+ // check that the number of containers in the first ResourceRequest in
+ // response for this sub-cluster matches expectations. -1 indicate the
+ // response should be null
+ private void checkExpectedAllocation(
+ Map<SubClusterId, List<ResourceRequest>> response, String subCluster,
+ long totResourceRequests, long totContainers) {
+ if (totContainers == -1) {
+ Assert.assertNull(response.get(SubClusterId.newInstance(subCluster)));
+ } else {
+ SubClusterId sc = SubClusterId.newInstance(subCluster);
+ Assert.assertEquals(totResourceRequests, response.get(sc).size());
+
+ long actualContCount = 0;
+ for (ResourceRequest rr : response.get(sc)) {
+ actualContCount += rr.getNumContainers();
+ }
+ Assert.assertEquals(totContainers, actualContCount);
+ }
+ }
+
+ private void validateSplit(Map<SubClusterId, List<ResourceRequest>> split,
+ List<ResourceRequest> original) throws YarnException {
+
+ SubClusterResolver resolver =
+ getFederationPolicyContext().getFederationSubclusterResolver();
+
+ // Apply general validation rules
+ int numUsedSubclusters = split.size();
+
+ Set<Long> originalIds = new HashSet<>();
+ Set<Long> splitIds = new HashSet<>();
+
+ int originalContainers = 0;
+ for (ResourceRequest rr : original) {
+ originalContainers += rr.getNumContainers();
+ originalIds.add(rr.getAllocationRequestId());
+ }
+
+ int splitContainers = 0;
+ for (Map.Entry<SubClusterId, List<ResourceRequest>> rrs : split
+ .entrySet()) {
+ for (ResourceRequest rr : rrs.getValue()) {
+ splitContainers += rr.getNumContainers();
+ splitIds.add(rr.getAllocationRequestId());
+ // check node-local asks are sent to right RM (only)
+ SubClusterId fid = null;
+ try {
+ fid = resolver.getSubClusterForNode(rr.getResourceName());
+ } catch (YarnException e) {
+ // ignore code will handle
+ }
+ if (!rrs.getKey().equals(getHomeSubCluster()) && fid != null
+ && !fid.equals(rrs.getKey())) {
+ Assert.fail("A node-local (or resolvable rack-local) RR should not "
+ + "be send to an RM other than what it resolves to.");
+ }
+ }
+ }
+
+ // check we are not inventing Allocation Ids
+ Assert.assertEquals(originalIds, splitIds);
+
+ // check we are not exceedingly replicating the container asks among
+ // RMs (a little is allowed due to rounding of fractional splits)
+ Assert.assertTrue(
+ " Containers requested (" + splitContainers + ") should "
+ + "not exceed the original count of containers ("
+ + originalContainers + ") by more than the number of subclusters ("
+ + numUsedSubclusters + ")",
+ originalContainers + numUsedSubclusters >= splitContainers);
+
+ // Test target Ids
+ for (SubClusterId targetId : split.keySet()) {
+ Assert.assertTrue("Target subclusters should be in the active set",
+ getActiveSubclusters().containsKey(targetId));
+ Assert.assertTrue(
+ "Target subclusters (" + targetId + ") should have weight >0 in "
+ + "the policy ",
+ getPolicyInfo().getRouterPolicyWeights()
+ .get(new SubClusterIdInfo(targetId)) > 0);
+ }
+ }
+
+ private void prettyPrintRequests(
+ Map<SubClusterId, List<ResourceRequest>> response) {
+ for (Map.Entry<SubClusterId, List<ResourceRequest>> entry : response
+ .entrySet()) {
+ String str = "";
+ for (ResourceRequest rr : entry.getValue()) {
+ str += " [id:" + rr.getAllocationRequestId() + " loc:"
+ + rr.getResourceName() + " numCont:" + rr.getNumContainers()
+ + "], ";
+ }
+ LOG.info(entry.getKey() + " --> " + str);
+ }
+ }
+
+ private List<ResourceRequest> createLargeRandomList(int numRR)
+ throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+ Random rand = new Random(1);
+ DefaultSubClusterResolverImpl resolver =
+ (DefaultSubClusterResolverImpl) getFederationPolicyContext()
+ .getFederationSubclusterResolver();
+
+ List<String> nodes =
+ new ArrayList<>(resolver.getNodeToSubCluster().keySet());
+
+ for (int i = 0; i < numRR; i++) {
+ String nodeName = nodes.get(rand.nextInt(nodes.size()));
+ long allocationId = (long) rand.nextInt(20);
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(allocationId,
+ nodeName, 1024, 1, 1, rand.nextInt(100), null, rand.nextBoolean()));
+ }
+ return out;
+ }
+
+ private List<ResourceRequest> createSimpleRequest() throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ ResourceRequest.ANY, 1024, 1, 1, 100, null, true));
+ return out;
+ }
+
+ private List<ResourceRequest> createZeroSizedANYRequest() throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ ResourceRequest.ANY, 1024, 1, 1, 0, null, true));
+ return out;
+ }
+
+ private List<ResourceRequest> createComplexRequest() throws Exception {
+
+ List<ResourceRequest> out = new ArrayList<>();
+
+ // create a single container request in sc0
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ "subcluster0-rack0-host0", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ "subcluster0-rack0", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(0L,
+ ResourceRequest.ANY, 1024, 1, 1, 1, null, false));
+
+ // create a single container request with 3 alternative hosts across sc1,sc2
+ // where we want 2 containers in sc1 and 1 in sc2
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster1-rack1-host1", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster1-rack1-host2", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster2-rack3-host3", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster1-rack1", 1024, 1, 1, 2, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ "subcluster2-rack3", 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(1L,
+ ResourceRequest.ANY, 1024, 1, 1, 2, null, false));
+
+ // create a non-local ANY request that can span anything
+ out.add(FederationPoliciesTestUtil.createResourceRequest(2L,
+ ResourceRequest.ANY, 1024, 1, 1, 100, null, true));
+
+ // create a single container request in sc0 with relaxed locality
+ out.add(FederationPoliciesTestUtil.createResourceRequest(3L,
+ "subcluster0-rack0-host0", 1024, 1, 1, 1, null, true));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(3L,
+ "subcluster0-rack0", 1024, 1, 1, 1, null, true));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(3L,
+ ResourceRequest.ANY, 1024, 1, 1, 1, null, true));
+
+ // create a request of an unknown node/rack and expect this to show up
+ // in homesubcluster
+ out.add(FederationPoliciesTestUtil.createResourceRequest(4L, "unknownNode",
+ 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(4L, "unknownRack",
+ 1024, 1, 1, 1, null, false));
+ out.add(FederationPoliciesTestUtil.createResourceRequest(4L,
+ ResourceRequest.ANY, 1024, 1, 1, 1, null, false));
+
+ return out;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index 9e94f72..906e35f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
@@ -29,12 +32,9 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-
/**
- * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the
- * load is properly considered for allocation.
+ * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load
+ * is properly considered for allocation.
*/
public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
@@ -47,12 +47,10 @@ public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
// simulate 20 active subclusters
for (int i = 0; i < 20; i++) {
- SubClusterIdInfo sc =
- new SubClusterIdInfo(String.format("sc%02d", i));
+ SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", i));
SubClusterInfo federationSubClusterInfo =
SubClusterInfo.newInstance(sc.toId(), null, null, null, null, -1,
- SubClusterState.SC_RUNNING, -1,
- generateClusterMetricsInfo(i));
+ SubClusterState.SC_RUNNING, -1, generateClusterMetricsInfo(i));
getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
float weight = getRand().nextInt(2);
if (i == 5) {
@@ -76,7 +74,7 @@ public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
private String generateClusterMetricsInfo(int id) {
long mem = 1024 * getRand().nextInt(277 * 100 - 1);
- //plant a best cluster
+ // plant a best cluster
if (id == 5) {
mem = 1024 * 277 * 100;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
index ff5175d..eefcfd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -16,6 +16,12 @@
*/
package org.apache.hadoop.yarn.server.federation.policies.router;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
@@ -28,12 +34,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
/**
* Simple test class for the {@link PriorityRouterPolicy}. Tests that the
* weights are correctly used for ordering the choice of sub-clusters.
@@ -72,8 +72,7 @@ public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest {
getPolicyInfo().setRouterPolicyWeights(routerWeights);
getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
- getPolicyInfo(),
- getActiveSubclusters());
+ getPolicyInfo(), getActiveSubclusters());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
index a612685..78967d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -17,6 +17,13 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
@@ -29,13 +36,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
/**
* Simple test class for the {@link WeightedRandomRouterPolicy}. Generate large
* number of randomized tests to check we are weighiting correctly even if
@@ -71,8 +71,7 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
- getPolicyInfo(),
- getActiveSubclusters());
+ getPolicyInfo(), getActiveSubclusters());
}
@@ -88,8 +87,8 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
float numberOfDraws = 1000000;
for (float i = 0; i < numberOfDraws; i++) {
- SubClusterId chosenId = ((FederationRouterPolicy) getPolicy()).
- getHomeSubcluster(getApplicationSubmissionContext());
+ SubClusterId chosenId = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
counter.get(chosenId).incrementAndGet();
}
@@ -113,13 +112,15 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
if (getActiveSubclusters().containsKey(counterEntry.getKey())) {
Assert.assertTrue(
"Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
- + " expected weight: " + expectedWeight, expectedWeight == 0 ||
- (actualWeight / expectedWeight) < 1.1
- && (actualWeight / expectedWeight) > 0.9);
+ + " expected weight: " + expectedWeight,
+ expectedWeight == 0 || (actualWeight / expectedWeight) < 1.1
+ && (actualWeight / expectedWeight) > 0.9);
} else {
- Assert.assertTrue(
- "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
- + " expected weight: " + expectedWeight, actualWeight == 0);
+ Assert
+ .assertTrue(
+ "Id " + counterEntry.getKey() + " Actual weight: "
+ + actualWeight + " expected weight: " + expectedWeight,
+ actualWeight == 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index f901329..87ed8d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.server.federation.utils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
@@ -26,6 +27,7 @@ import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolv
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.*;
+import org.apache.hadoop.yarn.util.Records;
import java.net.URL;
import java.nio.ByteBuffer;
@@ -48,6 +50,68 @@ public final class FederationPoliciesTestUtil {
// disabled.
}
+ private static final String FEDR_NODE_PREFIX = "fedr-test-node-";
+
+
+ public static List<ResourceRequest> createResourceRequests(String[] hosts,
+ int memory, int vCores, int priority, int containers,
+ String labelExpression, boolean relaxLocality) throws YarnException {
+ List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
+ for (String host : hosts) {
+ ResourceRequest hostReq =
+ createResourceRequest(host, memory, vCores, priority, containers,
+ labelExpression, relaxLocality);
+ reqs.add(hostReq);
+ ResourceRequest rackReq =
+ createResourceRequest("/default-rack", memory, vCores, priority,
+ containers, labelExpression, relaxLocality);
+ reqs.add(rackReq);
+ }
+
+ ResourceRequest offRackReq =
+ createResourceRequest(ResourceRequest.ANY, memory, vCores, priority,
+ containers, labelExpression, relaxLocality);
+ reqs.add(offRackReq);
+ return reqs;
+ }
+
+ protected static ResourceRequest createResourceRequest(String resource,
+ int memory, int vCores, int priority, int containers,
+ boolean relaxLocality) throws YarnException {
+ return createResourceRequest(resource, memory, vCores, priority, containers,
+ null, relaxLocality);
+ }
+
+ @SuppressWarnings("checkstyle:parameternumber")
+ public static ResourceRequest createResourceRequest(long id, String resource,
+ int memory, int vCores, int priority, int containers,
+ String labelExpression, boolean relaxLocality) throws YarnException {
+ ResourceRequest out =
+ createResourceRequest(resource, memory, vCores, priority, containers,
+ labelExpression, relaxLocality);
+ out.setAllocationRequestId(id);
+ return out;
+ }
+
+ public static ResourceRequest createResourceRequest(String resource,
+ int memory, int vCores, int priority, int containers,
+ String labelExpression, boolean relaxLocality) throws YarnException {
+ ResourceRequest req = Records.newRecord(ResourceRequest.class);
+ req.setResourceName(resource);
+ req.setNumContainers(containers);
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(priority);
+ req.setPriority(pri);
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemorySize(memory);
+ capability.setVirtualCores(vCores);
+ req.setCapability(capability);
+ if (labelExpression != null) {
+ req.setNodeLabelExpression(labelExpression);
+ }
+ req.setRelaxLocality(relaxLocality);
+ return req;
+ }
public static void initializePolicyContext(
FederationPolicyInitializationContext fpc, ConfigurableFederationPolicy
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1793757d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
index e4d6112..2b7e237 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
@@ -1,4 +1,8 @@
node1,subcluster1,rack1
node2 , subcluster2, RACK1
noDE3,subcluster3, rack2
-node4, subcluster3, rack2
\ No newline at end of file
+node4, subcluster3, rack2
+subcluster0-rack0-host0,subcluster0, subcluster0-rack0
+Subcluster1-RACK1-HOST1,subcluster1, subCluster1-RACK1
+SUBCLUSTER1-RACK1-HOST2,subcluster1, subCluster1-RACK1
+SubCluster2-RACK3-HOST3,subcluster2, subcluster2-rack3
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-5391. PolicyManager to tie
together Router/AMRM Federation policies. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5391. PolicyManager to tie together Router/AMRM Federation policies. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c834e70e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c834e70e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c834e70e
Branch: refs/heads/YARN-2915
Commit: c834e70ea17378b4d58740b51d6cfd6fc609671d
Parents: 1793757
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 1 19:54:18 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../policies/AbstractPolicyManager.java | 175 +++++++++++++++++++
.../FederationPolicyInitializationContext.java | 3 +-
.../policies/UniformBroadcastPolicyManager.java | 56 ++++++
.../policies/WeightedLocalityPolicyManager.java | 67 +++++++
.../records/SubClusterPolicyConfiguration.java | 13 ++
.../policies/BasePolicyManagerTest.java | 108 ++++++++++++
...ionPolicyInitializationContextValidator.java | 5 +-
.../TestUniformBroadcastPolicyManager.java | 40 +++++
.../TestWeightedLocalityPolicyManager.java | 79 +++++++++
.../utils/FederationPoliciesTestUtil.java | 2 +-
10 files changed, 545 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
new file mode 100644
index 0000000..e77f2e3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides basic implementation for common methods that multiple
+ * policies will need to implement.
+ */
+public abstract class AbstractPolicyManager implements
+ FederationPolicyManager {
+
+ private String queue;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class routerFederationPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class amrmProxyFederationPolicy;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AbstractPolicyManager.class);
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ if (amrmProxyFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The parameter "
+ + "amrmProxyFederationPolicy should be initialized in "
+ + this.getClass().getSimpleName() + " constructor.");
+ }
+
+ try {
+ return (FederationAMRMProxyPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, amrmProxyFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+
+ }
+
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationRouterPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+
+ public FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ //checks that sub-types properly initialize the types of policies
+ if (routerFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The policy "
+ + "type should be initialized in " + this.getClass().getSimpleName()
+ + " constructor.");
+ }
+
+ try {
+ return (FederationRouterPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, routerFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ @Override
+ public String getQueue() {
+ return queue;
+ }
+
+ @Override
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ /**
+ * Common functionality to instantiate a reinitialize a {@link
+ * ConfigurableFederationPolicy}.
+ */
+ private ConfigurableFederationPolicy internalPolicyGetter(
+ final FederationPolicyInitializationContext federationPolicyContext,
+ ConfigurableFederationPolicy oldInstance, Class policy)
+ throws FederationPolicyInitializationException {
+
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ if (oldInstance == null || !oldInstance.getClass().equals(policy)) {
+ try {
+ oldInstance = (ConfigurableFederationPolicy) policy.newInstance();
+ } catch (InstantiationException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (IllegalAccessException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ //copying the context to avoid side-effects
+ FederationPolicyInitializationContext modifiedContext =
+ updateContext(federationPolicyContext,
+ oldInstance.getClass().getCanonicalName());
+
+ oldInstance.reinitialize(modifiedContext);
+ return oldInstance;
+ }
+
+ /**
+ * This method is used to copy-on-write the context, that will be passed
+ * downstream to the router/amrmproxy policies.
+ */
+ private FederationPolicyInitializationContext updateContext(
+ FederationPolicyInitializationContext federationPolicyContext,
+ String type) {
+ // copying configuration and context to avoid modification of original
+ SubClusterPolicyConfiguration newConf = SubClusterPolicyConfiguration
+ .newInstance(federationPolicyContext
+ .getSubClusterPolicyConfiguration());
+ newConf.setType(type);
+
+ return new FederationPolicyInitializationContext(newConf,
+ federationPolicyContext.getFederationSubclusterResolver(),
+ federationPolicyContext.getFederationStateStoreFacade(),
+ federationPolicyContext.getHomeSubcluster());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
index 46dd6eb..4d29a41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -41,10 +41,11 @@ public class FederationPolicyInitializationContext {
public FederationPolicyInitializationContext(
SubClusterPolicyConfiguration policy, SubClusterResolver resolver,
- FederationStateStoreFacade storeFacade) {
+ FederationStateStoreFacade storeFacade, SubClusterId home) {
this.federationPolicyConfiguration = policy;
this.federationSubclusterResolver = resolver;
this.federationStateStoreFacade = storeFacade;
+ this.homeSubcluster = home;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..a01f8fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class represents a simple implementation of a {@code
+ * FederationPolicyManager}.
+ *
+ * It combines the basic policies: {@link UniformRandomRouterPolicy} and
+ * {@link BroadcastAMRMProxyPolicy}, which are designed to work together and
+ * "spread" the load among sub-clusters uniformly.
+ *
+ * This simple policy might impose heavy load on the RMs and return more
+ * containers than a job requested as all requests are (replicated and)
+ * broadcasted.
+ */
+public class UniformBroadcastPolicyManager
+ extends AbstractPolicyManager {
+
+ public UniformBroadcastPolicyManager() {
+ //this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = UniformRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = ByteBuffer.allocate(0);
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..f3c6673
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link WeightedRandomRouterPolicy} for the router and a {@link
+ * LocalityMulticastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class WeightedLocalityPolicyManager
+ extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public WeightedLocalityPolicyManager() {
+ //this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = WeightedRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(
+ WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
index 2839139..52807d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -40,6 +40,7 @@ import java.nio.ByteBuffer;
@Unstable
public abstract class SubClusterPolicyConfiguration {
+
@Private
@Unstable
public static SubClusterPolicyConfiguration newInstance(String queue,
@@ -52,6 +53,18 @@ public abstract class SubClusterPolicyConfiguration {
return policy;
}
+ @Private
+ @Unstable
+ public static SubClusterPolicyConfiguration newInstance(
+ SubClusterPolicyConfiguration conf) {
+ SubClusterPolicyConfiguration policy =
+ Records.newRecord(SubClusterPolicyConfiguration.class);
+ policy.setQueue(conf.getQueue());
+ policy.setType(conf.getType());
+ policy.setParams(conf.getParams());
+ return policy;
+ }
+
/**
* Get the name of the queue for which we are configuring a policy.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
new file mode 100644
index 0000000..c609886
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class provides common test methods for testing {@code
+ * FederationPolicyManager}s.
+ */
+public abstract class BasePolicyManagerTest {
+
+
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected FederationPolicyManager wfp = null;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedPolicyManager;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedAMRMProxyPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedRouterPolicy;
+
+
+ @Test
+ public void testSerializeAndInstantiate() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy,
+ expectedRouterPolicy);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testSerializeAndInstantiateBad1() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, String.class,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad2() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ String.class, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad3() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, String.class);
+ }
+
+ protected static void serializeAndDeserializePolicyManager(
+ FederationPolicyManager wfp, Class policyManagerType,
+ Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception {
+
+ // serializeConf it in a context
+ SubClusterPolicyConfiguration fpc =
+ wfp.serializeConf();
+ fpc.setType(policyManagerType.getCanonicalName());
+ FederationPolicyInitializationContext context = new
+ FederationPolicyInitializationContext();
+ context.setSubClusterPolicyConfiguration(fpc);
+ context
+ .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ context.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster"));
+
+ // based on the "context" created instantiate new class and use it
+ Class c = Class.forName(wfp.getClass().getCanonicalName());
+ FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance();
+
+ FederationAMRMProxyPolicy federationAMRMProxyPolicy =
+ wfp2.getAMRMPolicy(context, null);
+
+ //needed only for tests (getARMRMPolicy change the "type" in conf)
+ fpc.setType(wfp.getClass().getCanonicalName());
+
+ FederationRouterPolicy federationRouterPolicy =
+ wfp2.getRouterPolicy(context, null);
+
+ Assert.assertEquals(federationAMRMProxyPolicy.getClass(),
+ expAMRMProxyPolicy);
+
+ Assert.assertEquals(federationRouterPolicy.getClass(),
+ expRouterPolicy);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index c79fd2a..d906b92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMR
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
@@ -38,6 +39,7 @@ public class TestFederationPolicyInitializationContextValidator {
private SubClusterPolicyConfiguration goodConfig;
private SubClusterResolver goodSR;
private FederationStateStoreFacade goodFacade;
+ private SubClusterId goodHome;
private FederationPolicyInitializationContext context;
@Before
@@ -45,8 +47,9 @@ public class TestFederationPolicyInitializationContextValidator {
goodFacade = FederationPoliciesTestUtil.initFacade();
goodConfig = new MockPolicyManager().serializeConf();
goodSR = FederationPoliciesTestUtil.initResolver();
+ goodHome = SubClusterId.newInstance("homesubcluster");
context = new FederationPolicyInitializationContext(goodConfig, goodSR,
- goodFacade);
+ goodFacade, goodHome);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..542a5ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link UniformBroadcastPolicyManager}.
+ */
+public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ //config policy
+ wfp = new UniformBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = UniformBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = UniformRandomRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..ab9cec4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple test of {@link WeightedLocalityPolicyManager}.
+ */
+public class TestWeightedLocalityPolicyManager extends
+ BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new WeightedLocalityPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+ amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setAMRMPolicyWeights(amrmWeights);
+
+ ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo(
+ policyInfo);
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = WeightedLocalityPolicyManager.class;
+ expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ expectedRouterPolicy = WeightedRandomRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy,
+ expectedRouterPolicy);
+
+ //check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(((WeightedLocalityPolicyManager) wfp)
+ .getWeightedPolicyInfo(), policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c834e70e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index 87ed8d1..85fdc96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -143,7 +143,7 @@ public final class FederationPoliciesTestUtil {
SubClusterInfo> activeSubclusters) throws YarnException {
FederationPolicyInitializationContext context =
new FederationPolicyInitializationContext(null, initResolver(),
- initFacade());
+ initFacade(), SubClusterId.newInstance("homesubcluster"));
initializePolicyContext(context, policy, policyInfo, activeSubclusters);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-5634. Simplify
initialization/use of RouterPolicy via a RouterPolicyFacade. (Carlo Curino
via Subru).
Posted by su...@apache.org.
YARN-5634. Simplify initialization/use of RouterPolicy via a RouterPolicyFacade. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cc9d73d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cc9d73d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cc9d73d
Branch: refs/heads/YARN-2915
Commit: 6cc9d73df3bc2d15456458a2db92ff7c2298d305
Parents: c834e70
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Nov 16 19:39:25 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 9 +
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../yarn/conf/TestYarnConfigurationFields.java | 12 +
...ionPolicyInitializationContextValidator.java | 2 +-
.../PriorityBroadcastPolicyManager.java | 66 +++++
.../federation/policies/RouterPolicyFacade.java | 266 +++++++++++++++++++
.../policies/dao/WeightedPolicyInfo.java | 6 +-
.../utils/FederationStateStoreFacade.java | 16 +-
.../TestPriorityBroadcastPolicyManager.java | 72 +++++
.../policies/TestRouterPolicyFacade.java | 220 +++++++++++++++
.../utils/FederationStateStoreTestUtil.java | 22 +-
11 files changed, 693 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index bbd03a9..ee51094 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -310,6 +310,15 @@
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade"/>
+ <Or>
+ <Field name="globalConfMap"/>
+ <Field name="globalPolicyMap"/>
+ </Or>
+ <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+ </Match>
+
<!-- Don't care if putIfAbsent value is ignored -->
<Match>
<Package name="org.apache.hadoop.yarn.factories.impl.pb" />
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2836e50..26d21f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2523,6 +2523,19 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
+ public static final String DEFAULT_FEDERATION_POLICY_KEY = "*";
+
+ public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX
+ + "policy-manager";
+
+ public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
+ + ".hadoop.yarn.server.federation.policies.UniformBroadcastPolicyManager";
+
+ public static final String FEDERATION_POLICY_MANAGER_PARAMS =
+ FEDERATION_PREFIX + "policy-manager-params";
+
+ public static final String DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS = "";
+
////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3f3a06c..6e33c0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -78,6 +78,18 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_EPOCH);
+ // Federation policies configs to be ignored
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_POLICY_MANAGER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
configurationPropsToSkipCompare.add(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
index 1b83bbc..3c44e7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
@@ -57,7 +57,7 @@ public final class FederationPolicyInitializationContextValidator {
if (policyContext.getSubClusterPolicyConfiguration() == null) {
throw new FederationPolicyInitializationException(
- "The FederationSubclusterResolver provided is null. Cannot "
+ "The SubClusterPolicyConfiguration provided is null. Cannot "
+ "reinitalize successfully.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..ebdcf42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link PriorityRouterPolicy} for the router and a
+ * {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class PriorityBroadcastPolicyManager extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public PriorityBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = PriorityRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration.newInstance(getQueue(),
+ this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
new file mode 100644
index 0000000..a3fd15a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This class provides a facade to the policy subsystem, and handles the
+ * lifecycle of policies (e.g., refresh from remote, default behaviors etc.).
+ */
+public class RouterPolicyFacade {
+
+ private static final Log LOG =
+ LogFactory.getLog(LocalityMulticastAMRMProxyPolicy.class);
+
+ private final SubClusterResolver subClusterResolver;
+ private final FederationStateStoreFacade federationFacade;
+ private Map<String, SubClusterPolicyConfiguration> globalConfMap;
+
+ @VisibleForTesting
+ Map<String, FederationRouterPolicy> globalPolicyMap;
+
+ public RouterPolicyFacade(YarnConfiguration conf,
+ FederationStateStoreFacade facade, SubClusterResolver resolver,
+ SubClusterId homeSubcluster)
+ throws FederationPolicyInitializationException {
+
+ this.federationFacade = facade;
+ this.subClusterResolver = resolver;
+ this.globalConfMap = new ConcurrentHashMap<>();
+ this.globalPolicyMap = new ConcurrentHashMap<>();
+
+ // load default behavior from store if possible
+ String defaulKey = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+ SubClusterPolicyConfiguration configuration = null;
+ try {
+ configuration = federationFacade.getPolicyConfiguration(defaulKey);
+ } catch (YarnException e) {
+ LOG.warn("No fallback behavior defined in store, defaulting to XML "
+ + "configuration fallback behavior.");
+ }
+
+ // or from XML conf otherwise.
+ if (configuration == null) {
+ String defaultFederationPolicyManager =
+ conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER,
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER);
+ String defaultPolicyParamString =
+ conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS,
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+ ByteBuffer defaultPolicyParam = ByteBuffer
+ .wrap(defaultPolicyParamString.getBytes(StandardCharsets.UTF_8));
+
+ configuration = SubClusterPolicyConfiguration.newInstance(defaulKey,
+ defaultFederationPolicyManager, defaultPolicyParam);
+ }
+
+ // construct the required policy manager
+ FederationPolicyInitializationContext fallbackContext =
+ new FederationPolicyInitializationContext(configuration,
+ subClusterResolver, federationFacade, homeSubcluster);
+ FederationPolicyManager fallbackPolicyManager =
+ instantiatePolicyManager(configuration.getType());
+ fallbackPolicyManager.setQueue(defaulKey);
+
+ // add to the cache the fallback behavior
+ globalConfMap.put(defaulKey,
+ fallbackContext.getSubClusterPolicyConfiguration());
+ globalPolicyMap.put(defaulKey,
+ fallbackPolicyManager.getRouterPolicy(fallbackContext, null));
+
+ }
+
+ /**
+ * This method provides a wrapper of all policy functionalities for routing .
+ * Internally it manages configuration changes, and policy init/reinit.
+ *
+ * @param appSubmissionContext the application to route.
+ *
+ * @return the id of the subcluster that will be the "home" for this
+ * application.
+ *
+ * @throws YarnException if there are issues initializing policies, or no
+ * valid sub-cluster id could be found for this app.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+
+ // the maps are concurrent, but we need to protect from reset()
+ // reinitialization mid-execution by creating a new reference local to this
+ // method.
+ Map<String, SubClusterPolicyConfiguration> cachedConfs = globalConfMap;
+ Map<String, FederationRouterPolicy> policyMap = globalPolicyMap;
+
+ if (appSubmissionContext == null) {
+ throw new FederationPolicyException(
+ "The ApplicationSubmissionContext " + "cannot be null.");
+ }
+
+ String queue = appSubmissionContext.getQueue();
+
+ // respecting YARN behavior we assume default queue if the queue is not
+ // specified. This also ensures that "null" can be used as a key to get the
+ // default behavior.
+ if (queue == null) {
+ queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
+ }
+
+ // the facade might cache this request, based on its parameterization
+ SubClusterPolicyConfiguration configuration = null;
+
+ try {
+ configuration = federationFacade.getPolicyConfiguration(queue);
+ } catch (YarnException e) {
+ LOG.debug(e);
+ }
+
+ // If there is no policy configured for this queue, fallback to the baseline
+ // policy that is configured either in the store or via XML config (and
+ // cached)
+ if (configuration == null) {
+ try {
+ LOG.warn("There is no policies configured for queue: " + queue + " we"
+ + " fallback to default policy for: "
+ + YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+
+ queue = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+ configuration = federationFacade.getPolicyConfiguration(
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ } catch (YarnException e) {
+ // the fallback is not configure via store, but via XML, using
+ // previously loaded configuration.
+ configuration =
+ cachedConfs.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ }
+ }
+
+ // if the configuration has changed since last loaded, reinit the policy
+ // based on current configuration
+ if (!cachedConfs.containsKey(queue)
+ || !cachedConfs.get(queue).equals(configuration)) {
+ singlePolicyReinit(policyMap, cachedConfs, queue, configuration);
+ }
+
+ FederationRouterPolicy policy = policyMap.get(queue);
+ if (policy == null) {
+ // this should never happen, as the to maps are updated together
+ throw new FederationPolicyException("No FederationRouterPolicy found "
+ + "for queue: " + appSubmissionContext.getQueue() + " (for "
+ + "application: " + appSubmissionContext.getApplicationId() + ") "
+ + "and no default specified.");
+ }
+
+ return policy.getHomeSubcluster(appSubmissionContext);
+ }
+
+ /**
+ * This method reinitializes a policy and loads it in the policyMap.
+ *
+ * @param queue the queue to initialize a policy for.
+ * @param conf the configuration to use for initalization.
+ *
+ * @throws FederationPolicyInitializationException if initialization fails.
+ */
+ private void singlePolicyReinit(Map<String, FederationRouterPolicy> policyMap,
+ Map<String, SubClusterPolicyConfiguration> cachedConfs, String queue,
+ SubClusterPolicyConfiguration conf)
+ throws FederationPolicyInitializationException {
+
+ FederationPolicyInitializationContext context =
+ new FederationPolicyInitializationContext(conf, subClusterResolver,
+ federationFacade, null);
+ String newType = context.getSubClusterPolicyConfiguration().getType();
+ FederationRouterPolicy routerPolicy = policyMap.get(queue);
+
+ FederationPolicyManager federationPolicyManager =
+ instantiatePolicyManager(newType);
+ // set queue, reinit policy if required (implementation lazily check
+ // content of conf), and cache it
+ federationPolicyManager.setQueue(queue);
+ routerPolicy =
+ federationPolicyManager.getRouterPolicy(context, routerPolicy);
+
+ // we need the two put to be atomic (across multiple threads invoking
+ // this and reset operations)
+ synchronized (this) {
+ policyMap.put(queue, routerPolicy);
+ cachedConfs.put(queue, conf);
+ }
+ }
+
+ private static FederationPolicyManager instantiatePolicyManager(
+ String newType) throws FederationPolicyInitializationException {
+ FederationPolicyManager federationPolicyManager = null;
+ try {
+ // create policy instance and set queue
+ Class c = Class.forName(newType);
+ federationPolicyManager = (FederationPolicyManager) c.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (InstantiationException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (IllegalAccessException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ return federationPolicyManager;
+ }
+
+ /**
+ * This method flushes all cached configurations and policies. This should be
+ * invoked if the facade remains activity after very large churn of queues in
+ * the system.
+ */
+ public synchronized void reset() {
+
+ // remember the fallBack
+ SubClusterPolicyConfiguration conf =
+ globalConfMap.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+ FederationRouterPolicy policy =
+ globalPolicyMap.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+
+ globalConfMap = new ConcurrentHashMap<>();
+ globalPolicyMap = new ConcurrentHashMap<>();
+
+ // add to the cache a fallback with keyword null
+ globalConfMap.put(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY, conf);
+ globalPolicyMap.put(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY,
+ policy);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
index 62eb03b..e7b8afe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.federation.policies.dao;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
@@ -100,7 +100,7 @@ public class WeightedPolicyInfo {
JSONUnmarshaller unmarshaller = jsonjaxbContext.createJSONUnmarshaller();
final byte[] bytes = new byte[bb.remaining()];
bb.get(bytes);
- String params = new String(bytes, Charset.forName("UTF-8"));
+ String params = new String(bytes, StandardCharsets.UTF_8);
WeightedPolicyInfo weightedPolicyInfo = unmarshaller.unmarshalFromJSON(
new StringReader(params), WeightedPolicyInfo.class);
@@ -164,7 +164,7 @@ public class WeightedPolicyInfo {
}
try {
String s = toJSONString();
- return ByteBuffer.wrap(s.getBytes(Charset.forName("UTF-8")));
+ return ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8));
} catch (JAXBException j) {
throw new FederationPolicyInitializationException(j);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index 66a0b60..9b794de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoR
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -262,12 +263,17 @@ public final class FederationStateStoreFacade {
if (isCachingEnabled()) {
return getPoliciesConfigurations().get(queue);
} else {
- return stateStore
- .getPolicyConfiguration(
- GetSubClusterPolicyConfigurationRequest.newInstance(queue))
- .getPolicyConfiguration();
- }
+ GetSubClusterPolicyConfigurationResponse response =
+ stateStore.getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue));
+ if (response == null) {
+ throw new YarnException("The stateStore returned a null for "
+ + "GetSubClusterPolicyConfigurationResponse for queue " + queue);
+ } else {
+ return response.getPolicyConfiguration();
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..5e5bc83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test of {@link PriorityBroadcastPolicyManager}.
+ */
+public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new PriorityBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo);
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = PriorityBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = PriorityRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+
+ // check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(
+ ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(),
+ policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
new file mode 100644
index 0000000..4975a9f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test of {@link RouterPolicyFacade}.
+ */
+public class TestRouterPolicyFacade {
+
+ private RouterPolicyFacade routerFacade;
+ private List<SubClusterId> subClusterIds;
+ private FederationStateStore store;
+ private String queue1 = "queue1";
+ private String defQueueKey = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+
+ @Before
+ public void setup() throws YarnException {
+
+ // setting up a store and its facade (with caching off)
+ FederationStateStoreFacade fedFacade =
+ FederationStateStoreFacade.getInstance();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, "0");
+ store = new MemoryFederationStateStore();
+ store.init(conf);
+ fedFacade.reinitialize(store, conf);
+
+ FederationStateStoreTestUtil storeTestUtil =
+ new FederationStateStoreTestUtil(store);
+ storeTestUtil.registerSubClusters(10);
+
+ subClusterIds = storeTestUtil.getAllSubClusterIds(true);
+ store.setPolicyConfiguration(SetSubClusterPolicyConfigurationRequest
+ .newInstance(getUniformPolicy(queue1)));
+
+ SubClusterResolver resolver = FederationPoliciesTestUtil.initResolver();
+ routerFacade = new RouterPolicyFacade(new YarnConfiguration(), fedFacade,
+ resolver, subClusterIds.get(0));
+ }
+
+ @Test
+ public void testConfigurationUpdate() throws YarnException {
+
+ // in this test we see what happens when the configuration is changed
+ // between calls. We achieve this by changing what is in the store.
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ when(applicationSubmissionContext.getQueue()).thenReturn(queue1);
+
+ // first call runs using standard UniformRandomRouterPolicy
+ SubClusterId chosen =
+ routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertTrue(routerFacade.globalPolicyMap
+ .get(queue1) instanceof UniformRandomRouterPolicy);
+
+ // then the operator changes how queue1 is routed setting it to
+ // PriorityRouterPolicy with weights favoring the first subcluster in
+ // subClusterIds.
+ store.setPolicyConfiguration(SetSubClusterPolicyConfigurationRequest
+ .newInstance(getPriorityPolicy(queue1)));
+
+ // second call is routed by new policy PriorityRouterPolicy
+ chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(chosen.equals(subClusterIds.get(0)));
+ Assert.assertTrue(routerFacade.globalPolicyMap
+ .get(queue1) instanceof PriorityRouterPolicy);
+ }
+
+ @Test
+ public void testGetHomeSubcluster() throws YarnException {
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ when(applicationSubmissionContext.getQueue()).thenReturn(queue1);
+
+ // the facade only contains the fallback behavior
+ Assert.assertTrue(routerFacade.globalPolicyMap.containsKey(defQueueKey)
+ && routerFacade.globalPolicyMap.size() == 1);
+
+ // when invoked it returns the expected SubClusterId.
+ SubClusterId chosen =
+ routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+
+ // now the caching of policies must have added an entry for this queue
+ Assert.assertTrue(routerFacade.globalPolicyMap.size() == 2);
+
+ // after the facade is used the policyMap contains the expected policy type.
+ Assert.assertTrue(routerFacade.globalPolicyMap
+ .get(queue1) instanceof UniformRandomRouterPolicy);
+
+ // the facade is again empty after reset
+ routerFacade.reset();
+ // the facade only contains the fallback behavior
+ Assert.assertTrue(routerFacade.globalPolicyMap.containsKey(defQueueKey)
+ && routerFacade.globalPolicyMap.size() == 1);
+
+ }
+
+ @Test
+ public void testFallbacks() throws YarnException {
+
+ // this tests the behavior of the system when the queue requested is
+ // not configured (or null) and there is no default policy configured
+ // for DEFAULT_FEDERATION_POLICY_KEY (*). This is our second line of
+ // defense.
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+
+ // The facade answers also for non-initialized policies (using the
+ // defaultPolicy)
+ String uninitQueue = "non-initialized-queue";
+ when(applicationSubmissionContext.getQueue()).thenReturn(uninitQueue);
+ SubClusterId chosen =
+ routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue));
+
+ // empty string
+ when(applicationSubmissionContext.getQueue()).thenReturn("");
+ chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue));
+
+ // null queue also falls back to default
+ when(applicationSubmissionContext.getQueue()).thenReturn(null);
+ chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertTrue(subClusterIds.contains(chosen));
+ Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue));
+
+ }
+
+ public static SubClusterPolicyConfiguration getUniformPolicy(String queue)
+ throws FederationPolicyInitializationException {
+
+ // we go through standard lifecycle instantiating a policyManager and
+ // configuring it and serializing it to a conf.
+ UniformBroadcastPolicyManager wfp = new UniformBroadcastPolicyManager();
+ wfp.setQueue(queue);
+
+ SubClusterPolicyConfiguration fpc = wfp.serializeConf();
+
+ return fpc;
+ }
+
+ public SubClusterPolicyConfiguration getPriorityPolicy(String queue)
+ throws FederationPolicyInitializationException {
+
+ // we go through standard lifecycle instantiating a policyManager and
+ // configuring it and serializing it to a conf.
+ PriorityBroadcastPolicyManager wfp = new PriorityBroadcastPolicyManager();
+
+ // equal weight to all subcluster
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ for (SubClusterId s : subClusterIds) {
+ routerWeights.put(new SubClusterIdInfo(s), 0.9f / subClusterIds.size());
+ }
+
+ // beside the first one who gets more weight
+ SubClusterIdInfo favorite = new SubClusterIdInfo((subClusterIds.get(0)));
+ routerWeights.put(favorite, (0.1f + 0.9f / subClusterIds.size()));
+
+ WeightedPolicyInfo policyInfo = new WeightedPolicyInfo();
+ policyInfo.setRouterPolicyWeights(routerWeights);
+ wfp.setWeightedPolicyInfo(policyInfo);
+ wfp.setQueue(queue);
+
+ // serializeConf it in a context
+ SubClusterPolicyConfiguration fpc = wfp.serializeConf();
+
+ return fpc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cc9d73d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
index c179521..649a61b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.federation.utils;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -29,6 +31,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHome
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -62,8 +65,8 @@ public class FederationStateStoreTestUtil {
String webAppAddress = "1.2.3.4:4";
return SubClusterInfo.newInstance(subClusterId, amRMAddress,
- clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
- CLOCK.getTime(), "capability");
+ clientRMAddress, rmAdminAddress, webAppAddress,
+ SubClusterState.SC_RUNNING, CLOCK.getTime(), "capability");
}
private void registerSubCluster(SubClusterId subClusterId)
@@ -97,6 +100,21 @@ public class FederationStateStoreTestUtil {
}
}
+ public List<SubClusterId> getAllSubClusterIds(
+ boolean filterInactiveSubclusters) throws YarnException {
+
+ List<SubClusterInfo> infos = stateStore
+ .getSubClusters(
+ GetSubClustersInfoRequest.newInstance(filterInactiveSubclusters))
+ .getSubClusters();
+ List<SubClusterId> ids = new ArrayList<>();
+ for (SubClusterInfo s : infos) {
+ ids.add(s.getSubClusterId());
+ }
+
+ return ids;
+ }
+
private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
String policyType) {
return SubClusterPolicyConfiguration.newInstance(queueName, policyType,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-3662. Federation Membership
State Store internal APIs.
Posted by su...@apache.org.
YARN-3662. Federation Membership State Store internal APIs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2b59155
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2b59155
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2b59155
Branch: refs/heads/YARN-2915
Commit: e2b5915576fcd8130b5648f0cfa71da84534cbb2
Parents: 5235a68
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Jul 29 16:53:40 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../hadoop-yarn-server-common/pom.xml | 8 +
.../store/FederationMembershipStateStore.java | 126 +++++++++
.../server/federation/store/package-info.java | 17 ++
.../store/records/GetSubClusterInfoRequest.java | 62 +++++
.../records/GetSubClusterInfoResponse.java | 62 +++++
.../records/GetSubClustersInfoRequest.java | 66 +++++
.../records/GetSubClustersInfoResponse.java | 66 +++++
.../records/SubClusterDeregisterRequest.java | 89 +++++++
.../records/SubClusterDeregisterResponse.java | 42 +++
.../records/SubClusterHeartbeatRequest.java | 149 +++++++++++
.../records/SubClusterHeartbeatResponse.java | 45 ++++
.../federation/store/records/SubClusterId.java | 100 +++++++
.../store/records/SubClusterInfo.java | 263 ++++++++++++++++++
.../records/SubClusterRegisterRequest.java | 74 +++++
.../records/SubClusterRegisterResponse.java | 44 +++
.../store/records/SubClusterState.java | 60 +++++
.../impl/pb/GetSubClusterInfoRequestPBImpl.java | 125 +++++++++
.../pb/GetSubClusterInfoResponsePBImpl.java | 134 ++++++++++
.../pb/GetSubClustersInfoRequestPBImpl.java | 108 ++++++++
.../pb/GetSubClustersInfoResponsePBImpl.java | 184 +++++++++++++
.../pb/SubClusterDeregisterRequestPBImpl.java | 156 +++++++++++
.../pb/SubClusterDeregisterResponsePBImpl.java | 77 ++++++
.../pb/SubClusterHeartbeatRequestPBImpl.java | 192 +++++++++++++
.../pb/SubClusterHeartbeatResponsePBImpl.java | 77 ++++++
.../records/impl/pb/SubClusterIdPBImpl.java | 75 ++++++
.../records/impl/pb/SubClusterInfoPBImpl.java | 267 +++++++++++++++++++
.../pb/SubClusterRegisterRequestPBImpl.java | 134 ++++++++++
.../pb/SubClusterRegisterResponsePBImpl.java | 77 ++++++
.../store/records/impl/pb/package-info.java | 17 ++
.../federation/store/records/package-info.java | 17 ++
.../proto/yarn_server_federation_protos.proto | 93 +++++++
.../records/TestFederationProtocolRecords.java | 133 +++++++++
32 files changed, 3139 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index fc23af8..9cc3cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -58,6 +58,13 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -146,6 +153,7 @@
<include>yarn_server_common_protos.proto</include>
<include>yarn_server_common_service_protos.proto</include>
<include>yarn_server_common_service_protos.proto</include>
+ <include>yarn_server_federation_protos.proto</include>
<include>ResourceTracker.proto</include>
<include>SCMUploader.proto</include>
<include>collectornodemanager_protocol.proto</include>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
new file mode 100644
index 0000000..378eadc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+
+/**
+ * FederationMembershipStateStore maintains the state of all
+ * <em>subcluster(s)</em> as encapsulated by {@code SubClusterInfo} for all the
+ * subcluster(s) that are participating in federation.
+ */
+@Private
+@Unstable
+public interface FederationMembershipStateStore {
+
+ /**
+ * Get the {@link Version} of the underlying federation membership state
+ * store.
+ *
+ * @return the {@link Version} of the underlying federation membership state
+ * store
+ */
+ Version getMembershipStateStoreVersion();
+
+ /**
+ * Register a <em>subcluster</em> by publishing capabilities as represented by
+ * {@code SubClusterInfo} to indicate participation in federation. This is
+ * typically done during initialization or restart/failover of the
+ * subcluster's <code>ResourceManager</code>. Upon successful registration, an
+ * identifier for the <em>subcluster</em> which is unique across the federated
+ * cluster is returned. The identifier is static, i.e. preserved across
+ * restarts and failover.
+ *
+ * @param registerSubClusterRequest the capabilities of the subcluster that
+ * wants to participate in federation. The subcluster id is also
+ * specified in case registration is triggered by restart/failover
+ * @return response empty on successfully if registration was successful
+ * @throws YarnException if the request is invalid/fails
+ */
+ SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest registerSubClusterRequest) throws YarnException;
+
+ /**
+ * Deregister a <em>subcluster</em> identified by {@code SubClusterId} to
+ * change state in federation. This can be done to mark the sub cluster lost,
+ * deregistered, or decommissioned.
+ *
+ * @param subClusterDeregisterRequest - the request to deregister the
+ * sub-cluster from federation.
+ * @return response empty on successfully deregistering the subcluster state
+ * @throws YarnException if the request is invalid/fails
+ */
+ SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest subClusterDeregisterRequest)
+ throws YarnException;
+
+ /**
+ * Periodic heartbeat from a <code>ResourceManager</code> participating in
+ * federation to indicate liveliness. The heartbeat publishes the current
+ * capabilities as represented by {@code SubClusterInfo} of the subcluster.
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ *
+ * @param subClusterHeartbeatRequest the capabilities of the subcluster that
+ * wants to keep alive its participation in federation
+ * @return response currently empty on if heartbeat was successfully processed
+ * @throws YarnException if the request is invalid/fails
+ */
+ SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest subClusterHeartbeatRequest)
+ throws YarnException;
+
+ /**
+ * Get the membership information of <em>subcluster</em> as identified by
+ * {@code SubClusterId}. The membership information includes the cluster
+ * endpoint and current capabilities as represented by {@code SubClusterInfo}.
+ *
+ * @param subClusterRequest the subcluster whose information is required
+ * @return the {@code SubClusterInfo}
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest subClusterRequest) throws YarnException;
+
+ /**
+ * Get the membership information of all the <em>subclusters</em> that are
+ * currently participating in federation. The membership information includes
+ * the cluster endpoint and current capabilities as represented by
+ * {@code SubClusterInfo}.
+ *
+ * @param subClustersRequest request for sub-clusters information
+ * @return a map of {@code SubClusterInfo} keyed by the {@code SubClusterId}
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest subClustersRequest) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java
new file mode 100644
index 0000000..33179e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java
new file mode 100644
index 0000000..656dea9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoRequest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain information about a sub-cluster identified by its
+ * {@link SubClusterId}.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterInfoRequest {
+
+ @Private
+ @Unstable
+ public static GetSubClusterInfoRequest newInstance(
+ SubClusterId subClusterId) {
+ GetSubClusterInfoRequest subClusterRequest =
+ Records.newRecord(GetSubClusterInfoRequest.class);
+ subClusterRequest.setSubClusterId(subClusterId);
+ return subClusterRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @param subClusterId the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java
new file mode 100644
index 0000000..f7bc74d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterInfoResponse.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Response to a query with {@link SubClusterInfo} about a sub-cluster.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterInfoResponse {
+
+ @Private
+ @Unstable
+ public static GetSubClusterInfoResponse newInstance(
+ SubClusterInfo subClusterInfo) {
+ GetSubClusterInfoResponse registerSubClusterRequest =
+ Records.newRecord(GetSubClusterInfoResponse.class);
+ registerSubClusterRequest.setSubClusterInfo(subClusterInfo);
+ return registerSubClusterRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @return the information pertaining to the sub-cluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterInfo getSubClusterInfo();
+
+ /**
+ * Set the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @param subClusterInfo the information pertaining to the sub-cluster
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterInfo(SubClusterInfo subClusterInfo);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
new file mode 100644
index 0000000..3264d81
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoRequest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain information about all sub-clusters that are
+ * participating in federation.
+ */
+@Private
+@Unstable
+public abstract class GetSubClustersInfoRequest {
+
+ @Public
+ @Unstable
+ public static GetSubClustersInfoRequest newInstance(
+ boolean filterInactiveSubClusters) {
+ GetSubClustersInfoRequest request =
+ Records.newRecord(GetSubClustersInfoRequest.class);
+ request.setFilterInactiveSubClusters(filterInactiveSubClusters);
+ return request;
+ }
+
+ /**
+ * Get the flag that indicates whether only active sub-clusters should be
+ * returned.
+ *
+ * @return whether to filter out inactive sub-clusters
+ */
+ @Public
+ @Unstable
+ public abstract boolean getFilterInactiveSubClusters();
+
+ /**
+ * Set the flag that indicates whether only active sub-clusters should be
+ * returned.
+ *
+ * @param filterInactiveSubClusters whether to filter out inactive
+ * sub-clusters
+ */
+ @Public
+ @Unstable
+ public abstract void setFilterInactiveSubClusters(
+ boolean filterInactiveSubClusters);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
new file mode 100644
index 0000000..bcf75ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Response to a query with list of {@link SubClusterInfo} about all
+ * sub-clusters that are currently participating in Federation.
+ */
+@Private
+@Unstable
+public abstract class GetSubClustersInfoResponse {
+
+ @Public
+ @Unstable
+ public static GetSubClustersInfoResponse newInstance(
+ List<SubClusterInfo> subClusters) {
+ GetSubClustersInfoResponse subClusterInfos =
+ Records.newRecord(GetSubClustersInfoResponse.class);
+ subClusterInfos.setSubClusters(subClusters);
+ return subClusterInfos;
+ }
+
+ /**
+ * Get the list of {@link SubClusterInfo} representing the information about
+ * all sub-clusters that are currently participating in Federation.
+ *
+ * @return the list of {@link SubClusterInfo}
+ */
+ @Public
+ @Unstable
+ public abstract List<SubClusterInfo> getSubClusters();
+
+ /**
+ * Set the list of {@link SubClusterInfo} representing the information about
+ * all sub-clusters that are currently participating in Federation.
+ *
+ * @param subClusters the list of {@link SubClusterInfo}
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusters(List<SubClusterInfo> subClusters);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java
new file mode 100644
index 0000000..50a50a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterRequest.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent to set the state of a subcluster to either
+ * SC_DECOMMISSIONED, SC_LOST, or SC_DEREGISTERED.
+ *
+ * <p>
+ * The update includes details such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>{@link SubClusterState}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterDeregisterRequest {
+
+ @Private
+ @Unstable
+ public static SubClusterDeregisterRequest newInstance(
+ SubClusterId subClusterId, SubClusterState subClusterState) {
+ SubClusterDeregisterRequest registerRequest =
+ Records.newRecord(SubClusterDeregisterRequest.class);
+ registerRequest.setSubClusterId(subClusterId);
+ registerRequest.setState(subClusterState);
+ return registerRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @param subClusterId the subcluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+
+ /**
+ * Get the {@link SubClusterState} of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterState getState();
+
+ /**
+ * Set the {@link SubClusterState} of the subcluster.
+ *
+ * @param state the state of the subCluster
+ */
+ @Private
+ @Unstable
+ public abstract void setState(SubClusterState state);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java
new file mode 100644
index 0000000..74fe994
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterDeregisterResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SubClusterDeregisterResponse contains the answer from the {@code
+ * FederationMembershipStateStore} to a request to deregister the sub cluster.
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ */
+@Private
+@Unstable
+public abstract class SubClusterDeregisterResponse {
+
+ @Private
+ @Unstable
+ public static SubClusterDeregisterResponse newInstance() {
+ SubClusterDeregisterResponse response =
+ Records.newRecord(SubClusterDeregisterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java
new file mode 100644
index 0000000..3a07c18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatRequest.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterHeartbeatRequest is a report of the runtime information of the
+ * subcluster that is participating in federation.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>The URL of the subcluster</li>
+ * <li>The timestamp representing the last start time of the subCluster</li>
+ * <li>{@code FederationsubClusterState}</li>
+ * <li>The current capacity and utilization of the subCluster</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterHeartbeatRequest {
+
+ @Private
+ @Unstable
+ public static SubClusterHeartbeatRequest newInstance(
+ SubClusterId subClusterId, SubClusterState state, String capability) {
+ return newInstance(subClusterId, 0, state, capability);
+ }
+
+ @Private
+ @Unstable
+ public static SubClusterHeartbeatRequest newInstance(
+ SubClusterId subClusterId, long lastHeartBeat, SubClusterState state,
+ String capability) {
+ SubClusterHeartbeatRequest subClusterHeartbeatRequest =
+ Records.newRecord(SubClusterHeartbeatRequest.class);
+ subClusterHeartbeatRequest.setSubClusterId(subClusterId);
+ subClusterHeartbeatRequest.setLastHeartBeat(lastHeartBeat);
+ subClusterHeartbeatRequest.setState(state);
+ subClusterHeartbeatRequest.setCapability(capability);
+ return subClusterHeartbeatRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subCluster.
+ *
+ * @param subClusterId the subCluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+
+ /**
+ * Get the last heart beat time of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract long getLastHeartBeat();
+
+ /**
+ * Set the last heartbeat time of the subcluster.
+ *
+ * @param time the last heartbeat time of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setLastHeartBeat(long time);
+
+ /**
+ * Get the {@link SubClusterState} of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterState getState();
+
+ /**
+ * Set the {@link SubClusterState} of the subcluster.
+ *
+ * @param state the state of the subCluster
+ */
+ @Private
+ @Unstable
+ public abstract void setState(SubClusterState state);
+
+ /**
+ * Get the current capacity and utilization of the subcluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @return the current capacity and utilization of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract String getCapability();
+
+ /**
+ * Set the current capacity and utilization of the subCluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @param capability the current capacity and utilization of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setCapability(String capability);
+
+ @Override
+ public String toString() {
+ return "SubClusterHeartbeatRequest [getSubClusterId() = "
+ + getSubClusterId() + ", getState() = " + getState()
+ + ", getLastHeartBeat = " + getLastHeartBeat() + ", getCapability() = "
+ + getCapability() + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java
new file mode 100644
index 0000000..0b7fd8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterHeartbeatResponse.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SubClusterHeartbeatResponse contains the response from the {@code
+ * FederationMembershipStateStore} to a periodic heartbeat to indicate
+ * liveliness from a <code>ResourceManager</code> participating in federation.
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ * <p>
+ * NOTE: This can be extended to push down policies in future
+ */
+@Private
+@Unstable
+public abstract class SubClusterHeartbeatResponse {
+
+ @Private
+ @Unstable
+ public static SubClusterHeartbeatResponse newInstance() {
+ SubClusterHeartbeatResponse response =
+ Records.newRecord(SubClusterHeartbeatResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
new file mode 100644
index 0000000..fec967d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterId represents the <em>globally unique</em> identifier for a
+ * subcluster that is participating in federation.
+ *
+ * <p>
+ * The globally unique nature of the identifier is obtained from the
+ * <code>FederationMembershipStateStore</code> on initialization.
+ */
+@Private
+@Unstable
+public abstract class SubClusterId implements Comparable<SubClusterId> {
+
+ @Private
+ @Unstable
+ public static SubClusterId newInstance(String subClusterId) {
+ SubClusterId id = Records.newRecord(SubClusterId.class);
+ id.setId(subClusterId);
+ return id;
+ }
+
+ /**
+ * Get the string identifier of the <em>subcluster</em> which is unique across
+ * the federated cluster. The identifier is static, i.e. preserved across
+ * restarts and failover.
+ *
+ * @return unique identifier of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract String getId();
+
+ /**
+ * Set the string identifier of the <em>subcluster</em> which is unique across
+ * the federated cluster. The identifier is static, i.e. preserved across
+ * restarts and failover.
+ *
+ * @param subClusterId unique identifier of the subcluster
+ */
+ @Private
+ @Unstable
+ protected abstract void setId(String subClusterId);
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ SubClusterId other = (SubClusterId) obj;
+ return this.getId().equals(other.getId());
+ }
+
+ @Override
+ public int hashCode() {
+ return getId().hashCode();
+ }
+
+ @Override
+ public int compareTo(SubClusterId other) {
+ return getId().compareTo(other.getId());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getId());
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
new file mode 100644
index 0000000..f13c8f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterInfo is a report of the runtime information of the subcluster that
+ * is participating in federation.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>The URL of the subcluster</li>
+ * <li>The timestamp representing the last start time of the subCluster</li>
+ * <li>{@code FederationsubClusterState}</li>
+ * <li>The current capacity and utilization of the subCluster</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterInfo {
+
+ @Private
+ @Unstable
+ public static SubClusterInfo newInstance(SubClusterId subClusterId,
+ String amRMServiceAddress, String clientRMServiceAddress,
+ String rmAdminServiceAddress, String rmWebServiceAddress,
+ SubClusterState state, long lastStartTime, String capability) {
+ return newInstance(subClusterId, amRMServiceAddress, clientRMServiceAddress,
+ rmAdminServiceAddress, rmWebServiceAddress, 0, state, lastStartTime,
+ capability);
+ }
+
+ @Private
+ @Unstable
+ public static SubClusterInfo newInstance(SubClusterId subClusterId,
+ String amRMServiceAddress, String clientRMServiceAddress,
+ String rmAdminServiceAddress, String rmWebServiceAddress,
+ long lastHeartBeat, SubClusterState state, long lastStartTime,
+ String capability) {
+ SubClusterInfo subClusterInfo = Records.newRecord(SubClusterInfo.class);
+ subClusterInfo.setSubClusterId(subClusterId);
+ subClusterInfo.setAMRMServiceAddress(amRMServiceAddress);
+ subClusterInfo.setClientRMServiceAddress(clientRMServiceAddress);
+ subClusterInfo.setRMAdminServiceAddress(rmAdminServiceAddress);
+ subClusterInfo.setRMWebServiceAddress(rmWebServiceAddress);
+ subClusterInfo.setLastHeartBeat(lastHeartBeat);
+ subClusterInfo.setState(state);
+ subClusterInfo.setLastStartTime(lastStartTime);
+ subClusterInfo.setCapability(capability);
+ return subClusterInfo;
+ }
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the
+ * subcluster.
+ *
+ * @return the subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getSubClusterId();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the
+ * subCluster.
+ *
+ * @param subClusterId the subCluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setSubClusterId(SubClusterId subClusterId);
+
+ /**
+ * Get the URL of the AM-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @return the URL of the AM-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>
+ */
+ @Public
+ @Unstable
+ public abstract String getAMRMServiceAddress();
+
+ /**
+ * Set the URL of the AM-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @param amRMServiceAddress the URL of the AM-RM service endpoint of the
+ * subcluster <code>ResourceManager</code>
+ */
+ @Private
+ @Unstable
+ public abstract void setAMRMServiceAddress(String amRMServiceAddress);
+
+ /**
+ * Get the URL of the client-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @return the URL of the client-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>
+ */
+ @Public
+ @Unstable
+ public abstract String getClientRMServiceAddress();
+
+ /**
+ * Set the URL of the client-RM service endpoint of the subcluster
+ * <code>ResourceManager</code>.
+ *
+ * @param clientRMServiceAddress the URL of the client-RM service endpoint of
+ * the subCluster <code>ResourceManager</code>
+ */
+ @Private
+ @Unstable
+ public abstract void setClientRMServiceAddress(String clientRMServiceAddress);
+
+ /**
+ * Get the URL of the <code>ResourceManager</code> administration service.
+ *
+ * @return the URL of the <code>ResourceManager</code> administration service
+ */
+ @Public
+ @Unstable
+ public abstract String getRMAdminServiceAddress();
+
+ /**
+ * Set the URL of the <code>ResourceManager</code> administration service.
+ *
+ * @param rmAdminServiceAddress the URL of the <code>ResourceManager</code>
+ * administration service.
+ */
+ @Private
+ @Unstable
+ public abstract void setRMAdminServiceAddress(String rmAdminServiceAddress);
+
+ /**
+ * Get the URL of the <code>ResourceManager</code> web application interface.
+ *
+ * @return the URL of the <code>ResourceManager</code> web application
+ * interface.
+ */
+ @Public
+ @Unstable
+ public abstract String getRMWebServiceAddress();
+
+ /**
+ * Set the URL of the <code>ResourceManager</code> web application interface.
+ *
+ * @param rmWebServiceAddress the URL of the <code>ResourceManager</code> web
+ * application interface.
+ */
+ @Private
+ @Unstable
+ public abstract void setRMWebServiceAddress(String rmWebServiceAddress);
+
+ /**
+ * Get the last heart beat time of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract long getLastHeartBeat();
+
+ /**
+ * Set the last heartbeat time of the subcluster.
+ *
+ * @param time the last heartbeat time of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setLastHeartBeat(long time);
+
+ /**
+ * Get the {@link SubClusterState} of the subcluster.
+ *
+ * @return the state of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterState getState();
+
+ /**
+ * Set the {@link SubClusterState} of the subcluster.
+ *
+ * @param state the state of the subCluster
+ */
+ @Private
+ @Unstable
+ public abstract void setState(SubClusterState state);
+
+ /**
+ * Get the timestamp representing the last start time of the subcluster.
+ *
+ * @return the timestamp representing the last start time of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract long getLastStartTime();
+
+ /**
+ * Set the timestamp representing the last start time of the subcluster.
+ *
+ * @param lastStartTime the timestamp representing the last start time of the
+ * subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setLastStartTime(long lastStartTime);
+
+ /**
+ * Get the current capacity and utilization of the subcluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @return the current capacity and utilization of the subcluster
+ */
+ @Public
+ @Unstable
+ public abstract String getCapability();
+
+ /**
+ * Set the current capacity and utilization of the subCluster. This is the
+ * JAXB marshalled string representation of the <code>ClusterMetrics</code>.
+ *
+ * @param capability the current capacity and utilization of the subcluster
+ */
+ @Private
+ @Unstable
+ public abstract void setCapability(String capability);
+
+ @Override
+ public String toString() {
+ return "SubClusterInfo [getSubClusterId() = " + getSubClusterId()
+ + ", getAMRMServiceAddress() = " + getAMRMServiceAddress()
+ + ", getClientRMServiceAddress() = " + getClientRMServiceAddress()
+ + ", getRMAdminServiceAddress() = " + getRMAdminServiceAddress()
+ + ", getRMWebServiceAddress() = " + getRMWebServiceAddress()
+ + ", getState() = " + getState() + ", getLastStartTime() = "
+ + getLastStartTime() + ", getCapability() = " + getCapability() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java
new file mode 100644
index 0000000..8864fe3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterRequest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * SubClusterRegisterRequest is a request by a sub-cluster
+ * {@code ResourceManager} to participate in federation.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link SubClusterId}</li>
+ * <li>The URL of the subcluster</li>
+ * <li>The timestamp representing the last start time of the subCluster</li>
+ * <li>{@code FederationsubClusterState}</li>
+ * <li>The current capacity and utilization of the subCluster</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class SubClusterRegisterRequest {
+
+ @Private
+ @Unstable
+ public static SubClusterRegisterRequest newInstance(
+ SubClusterInfo subClusterInfo) {
+ SubClusterRegisterRequest registerSubClusterRequest =
+ Records.newRecord(SubClusterRegisterRequest.class);
+ registerSubClusterRequest.setSubClusterInfo(subClusterInfo);
+ return registerSubClusterRequest;
+ }
+
+ /**
+ * Get the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @return the information pertaining to the sub-cluster
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterInfo getSubClusterInfo();
+
+ /**
+ * Set the {@link SubClusterInfo} encapsulating the information about the
+ * sub-cluster.
+ *
+ * @param subClusterInfo the information pertaining to the sub-cluster
+ */
+ @Public
+ @Unstable
+ public abstract void setSubClusterInfo(SubClusterInfo subClusterInfo);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java
new file mode 100644
index 0000000..060a857
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterRegisterResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SubClusterRegisterResponse contains the response from the {@code
+ * FederationMembershipStateStore} to a registration request from a
+ * <code>ResourceManager</code> to participate in federation.
+ *
+ * Currently response is empty if the operation was successful, if not an
+ * exception reporting reason for a failure.
+ */
+@Private
+@Unstable
+public abstract class SubClusterRegisterResponse {
+
+ @Private
+ @Unstable
+ public static SubClusterRegisterResponse newInstance() {
+ SubClusterRegisterResponse response =
+ Records.newRecord(SubClusterRegisterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
new file mode 100644
index 0000000..22cec99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p>
+ * State of a <code>SubCluster</code>.
+ * </p>
+ */
+@Private
+@Unstable
+public enum SubClusterState {
+ /** Newly registered subcluster, before the first heartbeat. */
+ SC_NEW,
+
+ /** Subcluster is registered and the RM sent a heartbeat recently. */
+ SC_RUNNING,
+
+ /** Subcluster is unhealthy. */
+ SC_UNHEALTHY,
+
+ /** Subcluster is in the process of being out of service. */
+ SC_DECOMMISSIONING,
+
+ /** Subcluster is out of service. */
+ SC_DECOMMISSIONED,
+
+ /** RM has not sent a heartbeat for some configured time threshold. */
+ SC_LOST,
+
+ /** Subcluster has unregistered. */
+ SC_UNREGISTERED;
+
+ public boolean isUnusable() {
+ return (this != SC_RUNNING && this != SC_NEW);
+ }
+
+ public boolean isFinal() {
+ return (this == SC_UNREGISTERED || this == SC_DECOMMISSIONED
+ || this == SC_LOST);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java
new file mode 100644
index 0000000..c61c419
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoRequestPBImpl.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClusterInfoRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClusterInfoRequestPBImpl extends GetSubClusterInfoRequest {
+
+ private GetSubClusterInfoRequestProto proto =
+ GetSubClusterInfoRequestProto.getDefaultInstance();
+ private GetSubClusterInfoRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetSubClusterInfoRequestPBImpl() {
+ builder = GetSubClusterInfoRequestProto.newBuilder();
+ }
+
+ public GetSubClusterInfoRequestPBImpl(GetSubClusterInfoRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterInfoRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterInfoRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+ GetSubClusterInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasSubClusterId()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getSubClusterId());
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+ maybeInitBuilder();
+ if (subClusterId == null) {
+ builder.clearSubClusterId();
+ return;
+ }
+ builder.setSubClusterId(convertToProtoFormat(subClusterId));
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+ return new SubClusterIdPBImpl(sc);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+ return ((SubClusterIdPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
new file mode 100644
index 0000000..d0bcc33
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClusterInfoResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClusterInfoResponsePBImpl extends GetSubClusterInfoResponse {
+
+ private GetSubClusterInfoResponseProto proto =
+ GetSubClusterInfoResponseProto.getDefaultInstance();
+ private GetSubClusterInfoResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterInfo subClusterInfo = null;
+
+ public GetSubClusterInfoResponsePBImpl() {
+ builder = GetSubClusterInfoResponseProto.newBuilder();
+ }
+
+ public GetSubClusterInfoResponsePBImpl(GetSubClusterInfoResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterInfoResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterInfoResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterInfo != null) {
+ builder.setSubClusterInfo(convertToProtoFormat(this.subClusterInfo));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterInfo getSubClusterInfo() {
+ GetSubClusterInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.subClusterInfo != null) {
+ return this.subClusterInfo;
+ }
+ if (!p.hasSubClusterInfo()) {
+ return null;
+ }
+ this.subClusterInfo = convertFromProtoFormat(p.getSubClusterInfo());
+ return this.subClusterInfo;
+ }
+
+ @Override
+ public void setSubClusterInfo(SubClusterInfo subClusterInfo) {
+ maybeInitBuilder();
+ if (subClusterInfo == null) {
+ builder.clearSubClusterInfo();
+ }
+ this.subClusterInfo = subClusterInfo;
+ }
+
+ private SubClusterInfo convertFromProtoFormat(
+ SubClusterInfoProto clusterInfo) {
+ return new SubClusterInfoPBImpl(clusterInfo);
+ }
+
+ private SubClusterInfoProto convertToProtoFormat(SubClusterInfo clusterInfo) {
+ return ((SubClusterInfoPBImpl) clusterInfo).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java
new file mode 100644
index 0000000..2b848c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoRequestPBImpl.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClustersInfoRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClustersInfoRequestPBImpl extends GetSubClustersInfoRequest {
+
+ private GetSubClustersInfoRequestProto proto =
+ GetSubClustersInfoRequestProto.getDefaultInstance();
+ private GetSubClustersInfoRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetSubClustersInfoRequestPBImpl() {
+ builder = GetSubClustersInfoRequestProto.newBuilder();
+ }
+
+ public GetSubClustersInfoRequestPBImpl(GetSubClustersInfoRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClustersInfoRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClustersInfoRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public boolean getFilterInactiveSubClusters() {
+ GetSubClustersInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getFilterInactiveSubclusters();
+ }
+
+ @Override
+ public void setFilterInactiveSubClusters(boolean filterInactiveSubClusters) {
+ maybeInitBuilder();
+ builder.setFilterInactiveSubclusters(filterInactiveSubClusters);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2b59155/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
new file mode 100644
index 0000000..d39ef7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link GetSubClustersInfoResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClustersInfoResponsePBImpl
+ extends GetSubClustersInfoResponse {
+
+ private GetSubClustersInfoResponseProto proto =
+ GetSubClustersInfoResponseProto.getDefaultInstance();
+ private GetSubClustersInfoResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List<SubClusterInfo> subClusterInfos;
+
+ public GetSubClustersInfoResponsePBImpl() {
+ builder = GetSubClustersInfoResponseProto.newBuilder();
+ }
+
+ public GetSubClustersInfoResponsePBImpl(
+ GetSubClustersInfoResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClustersInfoResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterInfos != null) {
+ addReservationResourcesToProto();
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClustersInfoResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public List<SubClusterInfo> getSubClusters() {
+ initSubClustersInfoList();
+ return subClusterInfos;
+ }
+
+ @Override
+ public void setSubClusters(List<SubClusterInfo> subClusters) {
+ if (subClusters == null) {
+ builder.clearSubClusterInfos();
+ return;
+ }
+ this.subClusterInfos = subClusters;
+ }
+
+ private void initSubClustersInfoList() {
+ if (this.subClusterInfos != null) {
+ return;
+ }
+ GetSubClustersInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<SubClusterInfoProto> subClusterInfosList = p.getSubClusterInfosList();
+ subClusterInfos = new ArrayList<SubClusterInfo>();
+
+ for (SubClusterInfoProto r : subClusterInfosList) {
+ subClusterInfos.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addReservationResourcesToProto() {
+ maybeInitBuilder();
+ builder.clearSubClusterInfos();
+ if (subClusterInfos == null) {
+ return;
+ }
+ Iterable<SubClusterInfoProto> iterable =
+ new Iterable<SubClusterInfoProto>() {
+ @Override
+ public Iterator<SubClusterInfoProto> iterator() {
+ return new Iterator<SubClusterInfoProto>() {
+
+ private Iterator<SubClusterInfo> iter =
+ subClusterInfos.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public SubClusterInfoProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllSubClusterInfos(iterable);
+ }
+
+ private SubClusterInfoProto convertToProtoFormat(SubClusterInfo r) {
+ return ((SubClusterInfoPBImpl) r).getProto();
+ }
+
+ private SubClusterInfoPBImpl convertFromProtoFormat(SubClusterInfoProto r) {
+ return new SubClusterInfoPBImpl(r);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-6210. FairScheduler: Node
reservations can interfere with preemption. (kasha)
Posted by su...@apache.org.
YARN-6210. FairScheduler: Node reservations can interfere with preemption. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/718ad9f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/718ad9f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/718ad9f6
Branch: refs/heads/YARN-2915
Commit: 718ad9f6ee93d4145f2bb19b7582ce4e1174feaf
Parents: 732ee6f
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Wed Feb 22 15:45:45 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Wed Feb 22 15:46:07 2017 -0800
----------------------------------------------------------------------
.../resource/DefaultResourceCalculator.java | 3 +-
.../resource/DominantResourceCalculator.java | 13 +-
.../yarn/util/resource/ResourceCalculator.java | 32 ++++-
.../scheduler/fair/FSAppAttempt.java | 61 ++++++---
.../DominantResourceFairnessPolicy.java | 8 +-
.../fair/policies/FairSharePolicy.java | 3 +-
.../scheduler/fair/TestFairScheduler.java | 127 ++++++++-----------
.../fair/TestFairSchedulerPreemption.java | 44 +++++--
8 files changed, 180 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 42c45ad..ef7229c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -30,7 +30,8 @@ public class DefaultResourceCalculator extends ResourceCalculator {
LogFactory.getLog(DefaultResourceCalculator.class);
@Override
- public int compare(Resource unused, Resource lhs, Resource rhs) {
+ public int compare(Resource unused, Resource lhs, Resource rhs,
+ boolean singleType) {
// Only consider memory
return Long.compare(lhs.getMemorySize(), rhs.getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 9f1c8d7..032aa02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -51,17 +51,18 @@ public class DominantResourceCalculator extends ResourceCalculator {
LogFactory.getLog(DominantResourceCalculator.class);
@Override
- public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
+ public int compare(Resource clusterResource, Resource lhs, Resource rhs,
+ boolean singleType) {
if (lhs.equals(rhs)) {
return 0;
}
if (isInvalidDivisor(clusterResource)) {
- if ((lhs.getMemorySize() < rhs.getMemorySize() && lhs.getVirtualCores() > rhs
- .getVirtualCores())
- || (lhs.getMemorySize() > rhs.getMemorySize() && lhs.getVirtualCores() < rhs
- .getVirtualCores())) {
+ if ((lhs.getMemorySize() < rhs.getMemorySize() &&
+ lhs.getVirtualCores() > rhs.getVirtualCores()) ||
+ (lhs.getMemorySize() > rhs.getMemorySize() &&
+ lhs.getVirtualCores() < rhs.getVirtualCores())) {
return 0;
} else if (lhs.getMemorySize() > rhs.getMemorySize()
|| lhs.getVirtualCores() > rhs.getVirtualCores()) {
@@ -79,7 +80,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
return -1;
} else if (l > r) {
return 1;
- } else {
+ } else if (!singleType) {
l = getResourceAsValue(clusterResource, lhs, false);
r = getResourceAsValue(clusterResource, rhs, false);
if (l < r) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 50ce04c..a2f85b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -28,8 +28,36 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Unstable
public abstract class ResourceCalculator {
- public abstract int
- compare(Resource clusterResource, Resource lhs, Resource rhs);
+ /**
+ * On a cluster with capacity {@code clusterResource}, compare {@code lhs}
+ * and {@code rhs}. Consider all resources unless {@code singleType} is set
+ * to true. When {@code singleType} is set to true, consider only one
+ * resource as per the {@link ResourceCalculator} implementation; the
+ * {@link DefaultResourceCalculator} considers memory and
+ * {@link DominantResourceCalculator} considers the dominant resource.
+ *
+ * @param clusterResource cluster capacity
+ * @param lhs First {@link Resource} to compare
+ * @param rhs Second {@link Resource} to compare
+ * @param singleType Whether to consider a single resource type or all
+ * resource types
+ * @return -1 if {@code lhs} is smaller, 0 if equal and 1 if it is larger
+ */
+ public abstract int compare(
+ Resource clusterResource, Resource lhs, Resource rhs, boolean singleType);
+
+ /**
+ * On a cluster with capacity {@code clusterResource}, compare {@code lhs}
+ * and {@code rhs} considering all resources.
+ *
+ * @param clusterResource cluster capacity
+ * @param lhs First {@link Resource} to compare
+ * @param rhs Second {@link Resource} to compare
+ * @return -1 if {@code lhs} is smaller, 0 if equal and 1 if it is larger
+ */
+ public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
+ return compare(clusterResource, lhs, rhs, false);
+ }
public static int divideAndCeil(int a, int b) {
if (b == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 6ed0660..6c61b45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -605,8 +604,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resource usageAfterPreemption = Resources.subtract(
getResourceUsage(), container.getAllocatedResource());
- return !Resources.lessThan(fsQueue.getPolicy().getResourceCalculator(),
- scheduler.getClusterResource(), usageAfterPreemption, getFairShare());
+ return !isUsageBelowShare(usageAfterPreemption, getFairShare());
}
/**
@@ -833,9 +831,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
// The desired container won't fit here, so reserve
- if (isReservable(capability) && reserve(
- pendingAsk.getPerAllocationResource(), node, reservedContainer, type,
- schedulerKey)) {
+ if (isReservable(capability) &&
+ reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer,
+ type, schedulerKey)) {
if (isWaitingForAMContainer()) {
updateAMDiagnosticMsg(capability,
" exceed the available resources of the node and the request is"
@@ -857,8 +855,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
private boolean isReservable(Resource capacity) {
- return scheduler.isAtLeastReservationThreshold(
- getQueue().getPolicy().getResourceCalculator(), capacity);
+ // Reserve only when the app is starved and the requested container size
+ // is larger than the configured threshold
+ return isStarved() &&
+ scheduler.isAtLeastReservationThreshold(
+ getQueue().getPolicy().getResourceCalculator(), capacity);
}
/**
@@ -1089,34 +1090,51 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @return freshly computed fairshare starvation
*/
Resource fairShareStarvation() {
+ long now = scheduler.getClock().getTime();
Resource threshold = Resources.multiply(
getFairShare(), fsQueue.getFairSharePreemptionThreshold());
- Resource starvation = Resources.componentwiseMin(threshold, demand);
- Resources.subtractFromNonNegative(starvation, getResourceUsage());
+ Resource fairDemand = Resources.componentwiseMin(threshold, demand);
- long now = scheduler.getClock().getTime();
- boolean starved = !Resources.isNone(starvation);
+ // Check if the queue is starved for fairshare
+ boolean starved = isUsageBelowShare(getResourceUsage(), fairDemand);
if (!starved) {
lastTimeAtFairShare = now;
}
- if (starved &&
- (now - lastTimeAtFairShare > fsQueue.getFairSharePreemptionTimeout())) {
- this.fairshareStarvation = starvation;
+ if (!starved ||
+ now - lastTimeAtFairShare < fsQueue.getFairSharePreemptionTimeout()) {
+ fairshareStarvation = Resources.none();
} else {
- this.fairshareStarvation = Resources.none();
+ // The app has been starved for longer than preemption-timeout.
+ fairshareStarvation =
+ Resources.subtractFromNonNegative(fairDemand, getResourceUsage());
}
- return this.fairshareStarvation;
+ return fairshareStarvation;
+ }
+
+ /**
+ * Helper method that checks if {@code usage} is strictly less than
+ * {@code share}.
+ */
+ private boolean isUsageBelowShare(Resource usage, Resource share) {
+ return fsQueue.getPolicy().getResourceCalculator().compare(
+ scheduler.getClusterResource(), usage, share, true) < 0;
}
/**
* Helper method that captures if this app is identified to be starved.
* @return true if the app is starved for fairshare, false otherwise
*/
- @VisibleForTesting
boolean isStarvedForFairShare() {
- return !Resources.isNone(fairshareStarvation);
+ return isUsageBelowShare(getResourceUsage(), getFairShare());
+ }
+
+ /**
+ * Is application starved for fairshare or minshare
+ */
+ private boolean isStarved() {
+ return isStarvedForFairShare() || !Resources.isNone(minshareStarvation);
}
/**
@@ -1333,6 +1351,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
@Override
+ public String toString() {
+ return getApplicationAttemptId() + " Alloc: " + getCurrentConsumption();
+ }
+
+ @Override
public boolean isPreemptable() {
return getQueue().isPreemptable();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
index 6f04cb7..369b8a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
@@ -155,8 +155,12 @@ public class DominantResourceFairnessPolicy extends SchedulingPolicy {
resourceOrder1, resourceOrder2);
}
if (res == 0) {
- // Apps are tied in fairness ratio. Break the tie by submit time.
- res = (int)(s1.getStartTime() - s2.getStartTime());
+ // Apps are tied in fairness ratio. Break the tie by submit time and job
+ // name to get a deterministic ordering, which is useful for unit tests.
+ res = (int) Math.signum(s1.getStartTime() - s2.getStartTime());
+ if (res == 0) {
+ res = s1.getName().compareTo(s2.getName());
+ }
}
return res;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
index 9036a03..f8cdb45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
@@ -131,8 +131,9 @@ public class FairSharePolicy extends SchedulingPolicy {
// Apps are tied in fairness ratio. Break the tie by submit time and job
// name to get a deterministic ordering, which is useful for unit tests.
res = (int) Math.signum(s1.getStartTime() - s2.getStartTime());
- if (res == 0)
+ if (res == 0) {
res = s1.getName().compareTo(s2.getName());
+ }
}
return res;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 0c3a614..4def53f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -119,6 +119,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mock;
import org.mockito.Mockito;
import org.xml.sax.SAXException;
@@ -2627,71 +2628,57 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(1, scheduler.getSchedulerApp(attId4).getLiveContainers().size());
}
+ /**
+ * Reserve at a lower priority and verify the lower priority request gets
+ * allocated
+ */
@Test (timeout = 5000)
- public void testReservationWhileMultiplePriorities() throws IOException {
+ public void testReservationWithMultiplePriorities() throws IOException {
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
// Add a node
- RMNode node1 =
- MockNodes
- .newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1");
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
-
- ApplicationAttemptId attId = createSchedulingRequest(1024, 4, "queue1",
- "user1", 1, 2);
- scheduler.update();
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
- scheduler.handle(updateEvent);
-
- FSAppAttempt app = scheduler.getSchedulerApp(attId);
- assertEquals(1, app.getLiveContainers().size());
-
- ContainerId containerId = scheduler.getSchedulerApp(attId)
- .getLiveContainers().iterator().next().getContainerId();
- // Cause reservation to be created
- createSchedulingRequestExistingApplication(1024, 4, 2, attId);
+ // Create first app and take up half resources so the second app that asks
+ // for the entire node won't have enough.
+ FSAppAttempt app1 = scheduler.getSchedulerApp(
+ createSchedulingRequest(1024, 1, "queue", "user", 1));
scheduler.update();
scheduler.handle(updateEvent);
+ assertEquals("Basic allocation failed", 1, app1.getLiveContainers().size());
- assertEquals(1, app.getLiveContainers().size());
- assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
- assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
-
- // Create request at higher priority
- createSchedulingRequestExistingApplication(1024, 4, 1, attId);
+ // Create another app and reserve at a lower priority first
+ ApplicationAttemptId attId =
+ createSchedulingRequest(2048, 2, "queue1", "user1", 1, 2);
+ FSAppAttempt app2 = scheduler.getSchedulerApp(attId);
scheduler.update();
scheduler.handle(updateEvent);
-
- assertEquals(1, app.getLiveContainers().size());
- // Reserved container should still be at lower priority
- for (RMContainer container : app.getReservedContainers()) {
- assertEquals(2,
- container.getReservedSchedulerKey().getPriority().getPriority());
- }
-
- // Complete container
- scheduler.allocate(attId, new ArrayList<ResourceRequest>(),
+ assertEquals("Reservation at lower priority failed",
+ 1, app2.getReservedContainers().size());
+
+ // Request container on the second app at a higher priority
+ createSchedulingRequestExistingApplication(2048, 2, 1, attId);
+
+ // Complete the first container so we can trigger allocation for app2
+ ContainerId containerId =
+ app1.getLiveContainers().iterator().next().getContainerId();
+ scheduler.allocate(app1.getApplicationAttemptId(), new ArrayList<>(),
Arrays.asList(containerId), null, null, NULL_UPDATE_REQUESTS);
- assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
- assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
-
- // Schedule at opening
- scheduler.update();
+
+ // Trigger allocation for app2
scheduler.handle(updateEvent);
-
+
// Reserved container (at lower priority) should be run
- Collection<RMContainer> liveContainers = app.getLiveContainers();
- assertEquals(1, liveContainers.size());
- for (RMContainer liveContainer : liveContainers) {
- Assert.assertEquals(2, liveContainer.getContainer().getPriority()
- .getPriority());
- }
- assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
- assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
+ Collection<RMContainer> liveContainers = app2.getLiveContainers();
+ assertEquals("Allocation post completion failed", 1, liveContainers.size());
+ assertEquals("High prio container allocated against low prio reservation",
+ 2, liveContainers.iterator().next().getContainer().
+ getPriority().getPriority());
}
@Test
@@ -3222,8 +3209,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
/**
- * If we update our ask to strictly request a node, it doesn't make sense to keep
- * a reservation on another.
+ * Strict locality requests shouldn't reserve resources on another node.
*/
@Test
public void testReservationsStrictLocality() throws IOException {
@@ -3231,40 +3217,39 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
- RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
- RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 2, "127.0.0.2");
+ // Add two nodes
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 1));
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 1));
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
- ApplicationAttemptId attId = createSchedulingRequest(1024, "queue1",
- "user1", 0);
+ // Submit application without container requests
+ ApplicationAttemptId attId =
+ createSchedulingRequest(1024, "queue1", "user1", 0);
FSAppAttempt app = scheduler.getSchedulerApp(attId);
-
- ResourceRequest nodeRequest = createResourceRequest(1024, node2.getHostName(), 1, 2, true);
- ResourceRequest rackRequest = createResourceRequest(1024, "rack1", 1, 2, true);
- ResourceRequest anyRequest = createResourceRequest(1024, ResourceRequest.ANY,
- 1, 2, false);
+
+ // Request a container on node2
+ ResourceRequest nodeRequest =
+ createResourceRequest(1024, node2.getHostName(), 1, 1, true);
+ ResourceRequest rackRequest =
+ createResourceRequest(1024, "rack1", 1, 1, false);
+ ResourceRequest anyRequest =
+ createResourceRequest(1024, ResourceRequest.ANY, 1, 1, false);
createSchedulingRequestExistingApplication(nodeRequest, attId);
createSchedulingRequestExistingApplication(rackRequest, attId);
createSchedulingRequestExistingApplication(anyRequest, attId);
-
scheduler.update();
+ // Heartbeat from node1. App shouldn't get an allocation or reservation
NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(nodeUpdateEvent);
- assertEquals(1, app.getLiveContainers().size());
- scheduler.handle(nodeUpdateEvent);
- assertEquals(1, app.getReservedContainers().size());
-
- // now, make our request node-specific (on a different node)
- rackRequest = createResourceRequest(1024, "rack1", 1, 1, false);
- anyRequest = createResourceRequest(1024, ResourceRequest.ANY,
- 1, 1, false);
- scheduler.allocate(attId, Arrays.asList(rackRequest, anyRequest),
- new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
-
+ assertEquals("App assigned a container on the wrong node",
+ 0, app.getLiveContainers().size());
scheduler.handle(nodeUpdateEvent);
- assertEquals(0, app.getReservedContainers().size());
+ assertEquals("App reserved a container on the wrong node",
+ 0, app.getReservedContainers().size());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/718ad9f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 480a329..322ad5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -72,7 +72,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
{"MinSharePreemptionWithDRF", 1},
{"FairSharePreemption", 2},
{"FairSharePreemptionWithDRF", 3}
- });
+ });
}
public TestFairSchedulerPreemption(String name, int mode)
@@ -110,6 +110,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
* |--- preemptable
* |--- child-1
* |--- child-2
+ * |--- preemptable-sibling
* |--- nonpreemptible
* |--- child-1
* |--- child-2
@@ -133,6 +134,10 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
out.println("</queue>"); // end of preemptable queue
+ out.println("<queue name=\"preemptable-sibling\">");
+ writePreemptionParams(out);
+ out.println("</queue>");
+
// Queue with preemption disallowed
out.println("<queue name=\"nonpreemptable\">");
out.println("<allowPreemptionFrom>false" +
@@ -269,10 +274,11 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
preemptHalfResources(queue2);
}
- private void verifyPreemption() throws InterruptedException {
+ private void verifyPreemption(int numStarvedAppContainers)
+ throws InterruptedException {
// Sleep long enough for four containers to be preempted.
for (int i = 0; i < 1000; i++) {
- if (greedyApp.getLiveContainers().size() == 4) {
+ if (greedyApp.getLiveContainers().size() == 2 * numStarvedAppContainers) {
break;
}
Thread.sleep(10);
@@ -280,13 +286,13 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
// Verify the right amount of containers are preempted from greedyApp
assertEquals("Incorrect number of containers on the greedy app",
- 4, greedyApp.getLiveContainers().size());
+ 2 * numStarvedAppContainers, greedyApp.getLiveContainers().size());
sendEnoughNodeUpdatesToAssignFully();
// Verify the preempted containers are assigned to starvingApp
assertEquals("Starved app is not assigned the right number of containers",
- 2, starvingApp.getLiveContainers().size());
+ numStarvedAppContainers, starvingApp.getLiveContainers().size());
}
private void verifyNoPreemption() throws InterruptedException {
@@ -305,7 +311,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
String queue = "root.preemptable.child-1";
submitApps(queue, queue);
if (fairsharePreemption) {
- verifyPreemption();
+ verifyPreemption(2);
} else {
verifyNoPreemption();
}
@@ -314,13 +320,13 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
@Test
public void testPreemptionBetweenTwoSiblingLeafQueues() throws Exception {
submitApps("root.preemptable.child-1", "root.preemptable.child-2");
- verifyPreemption();
+ verifyPreemption(2);
}
@Test
public void testPreemptionBetweenNonSiblingQueues() throws Exception {
submitApps("root.preemptable.child-1", "root.nonpreemptable.child-1");
- verifyPreemption();
+ verifyPreemption(2);
}
@Test
@@ -354,7 +360,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
setNumAMContainersPerNode(2);
preemptHalfResources("root.preemptable.child-2");
- verifyPreemption();
+ verifyPreemption(2);
ArrayList<RMContainer> containers =
(ArrayList<RMContainer>) starvingApp.getLiveContainers();
@@ -365,4 +371,24 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
assertTrue("Preempted containers should come from two different "
+ "nodes.", !host0.equals(host1));
}
+
+ @Test
+ public void testPreemptionBetweenSiblingQueuesWithParentAtFairShare()
+ throws InterruptedException {
+ // Run this test only for fairshare preemption
+ if (!fairsharePreemption) {
+ return;
+ }
+
+ // Let one of the child queues take over the entire cluster
+ takeAllResources("root.preemptable.child-1");
+
+ // Submit a job so half the resources go to parent's sibling
+ preemptHalfResources("root.preemptable-sibling");
+ verifyPreemption(2);
+
+ // Submit a job to the child's sibling to force preemption from the child
+ preemptHalfResources("root.preemptable.child-2");
+ verifyPreemption(1);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-5324. Stateless Federation
router policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5324. Stateless Federation router policies implementation. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f03e0f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f03e0f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f03e0f4
Branch: refs/heads/YARN-2915
Commit: 5f03e0f4936df1b87d36990e5018749e435e3f6a
Parents: 088ed95
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Sep 22 17:06:57 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../policies/FederationPolicyConfigurator.java | 91 -------
.../FederationPolicyInitializationContext.java | 11 +-
.../policies/FederationPolicyManager.java | 126 +++++++++
.../policies/FederationPolicyWriter.java | 45 ----
.../policies/dao/WeightedPolicyInfo.java | 253 +++++++++++++++++++
.../federation/policies/dao/package-info.java | 20 ++
.../router/BaseWeightedRouterPolicy.java | 150 +++++++++++
.../policies/router/LoadBasedRouterPolicy.java | 109 ++++++++
.../policies/router/PriorityRouterPolicy.java | 66 +++++
.../router/UniformRandomRouterPolicy.java | 85 +++++++
.../router/WeightedRandomRouterPolicy.java | 79 ++++++
.../store/records/SubClusterIdInfo.java | 75 ++++++
.../policies/BaseFederationPoliciesTest.java | 155 ++++++++++++
...ionPolicyInitializationContextValidator.java | 17 +-
.../router/TestLoadBasedRouterPolicy.java | 109 ++++++++
.../router/TestPriorityRouterPolicy.java | 87 +++++++
.../router/TestUniformRandomRouterPolicy.java | 65 +++++
.../router/TestWeightedRandomRouterPolicy.java | 127 ++++++++++
.../utils/FederationPoliciesTestUtil.java | 82 +++++-
19 files changed, 1604 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
deleted file mode 100644
index fdc3857..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-
-
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-
-import org.apache.hadoop.yarn.server.federation.policies.router
- .FederationRouterPolicy;
-
-/**
- * Implementors of this interface are capable to instantiate and (re)initalize
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} based on
- * a {@link FederationPolicyInitializationContext}. The reason to bind these two
- * policies together is to make sure we remain consistent across the router and
- * amrmproxy policy decisions.
- */
-public interface FederationPolicyConfigurator {
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
- * current context, otherwise a new instance initialized with the current
- * context is provided. If the instance is compatible with the current class
- * the implementors should attempt to reinitalize (retaining state). To affect
- * a complete policy reset oldInstance should be null.
- *
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
- *
- * @return an updated {@link FederationAMRMProxyPolicy
- }.
- *
- * @throws FederationPolicyInitializationException if the initialization
- * cannot be completed
- * properly. The oldInstance
- * should be still valid in
- * case of failed
- * initialization.
- */
- FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
- FederationAMRMProxyPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationRouterPolicy} reinitialized with the current
- * context, otherwise a new instance initialized with the current context is
- * provided. If the instance is compatible with the current class the
- * implementors should attempt to reinitalize (retaining state). To affect a
- * complete policy reset oldInstance shoulb be set to null.
- *
- * @param federationPolicyInitializationContext the current context
- * @param oldInstance the existing (possibly null)
- * instance.
- *
- * @return an updated {@link FederationRouterPolicy}.
- *
- * @throws FederationPolicyInitializationException if the initalization cannot
- * be completed properly. The
- * oldInstance should be still
- * valid in case of failed
- * initialization.
- */
- FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext
- federationPolicyInitializationContext,
- FederationRouterPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
index 879ccee..9347fd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -59,13 +59,12 @@ public class FederationPolicyInitializationContext {
/**
* Setter for the {@link SubClusterPolicyConfiguration}.
*
- * @param federationPolicyConfiguration the
- * {@link SubClusterPolicyConfiguration}
- * to be used for initialization.
+ * @param fedPolicyConfiguration the {@link SubClusterPolicyConfiguration}
+ * to be used for initialization.
*/
- public void setFederationPolicyConfiguration(
- SubClusterPolicyConfiguration federationPolicyConfiguration) {
- this.federationPolicyConfiguration = federationPolicyConfiguration;
+ public void setSubClusterPolicyConfiguration(
+ SubClusterPolicyConfiguration fedPolicyConfiguration) {
+ this.federationPolicyConfiguration = fedPolicyConfiguration;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
new file mode 100644
index 0000000..e5dba63
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+/**
+ *
+ * Implementors need to provide the ability to serliaze a policy and its
+ * configuration as a {@link SubClusterPolicyConfiguration}, as well as
+ * provide (re)initialization mechanics for the underlying
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
+ *
+ * The serialization aspects are used by admin APIs or a policy engine to
+ * store a serialized configuration in the {@code FederationStateStore},
+ * while the getters methods are used to obtain a propertly inizialized
+ * policy in the {@code Router} and {@code AMRMProxy} respectively.
+ *
+ * This interface by design binds together
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} and
+ * provide lifecycle support for serialization and deserialization, to reduce
+ * configuration mistakes (combining incompatible policies).
+ *
+ */
+public interface FederationPolicyManager {
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
+ * current context, otherwise a new instance initialized with the current
+ * context is provided. If the instance is compatible with the current class
+ * the implementors should attempt to reinitalize (retaining state). To affect
+ * a complete policy reset oldInstance should be null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationAMRMProxyPolicy
+ }.
+ *
+ * @throws FederationPolicyInitializationException if the initialization
+ * cannot be completed
+ * properly. The oldInstance
+ * should be still valid in
+ * case of failed
+ * initialization.
+ */
+ FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationRouterPolicy} reinitialized with the current
+ * context, otherwise a new instance initialized with the current context is
+ * provided. If the instance is compatible with the current class the
+ * implementors should attempt to reinitalize (retaining state). To affect a
+ * complete policy reset oldInstance shoulb be set to null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationRouterPolicy}.
+ *
+ * @throws FederationPolicyInitializationException if the initalization cannot
+ * be completed properly. The
+ * oldInstance should be still
+ * valid in case of failed
+ * initialization.
+ */
+ FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
+ * This is to be used when writing a policy object in the federation policy
+ * store.
+ *
+ * @return a valid policy configuration representing this object
+ * parametrization.
+ *
+ * @throws FederationPolicyInitializationException if the current state cannot
+ * be serialized properly
+ */
+ SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException;
+
+
+ /**
+ * This method returns the queue this policy is configured for.
+ * @return the name of the queue.
+ */
+ String getQueue();
+
+ /**
+ * This methods provides a setter for the queue this policy is specified for.
+ * @param queue the name of the queue.
+ */
+ void setQueue(String queue);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
deleted file mode 100644
index 5034b7e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-/**
- * Implementors of this class are able to serializeConf the configuraiton of a
- * policy as a {@link SubClusterPolicyConfiguration}. This is used during the
- * lifetime of a policy from the admin APIs or policy engine to serializeConf
- * the policy into the policy store.
- */
-public interface FederationPolicyWriter {
-
- /**
- /**
- * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
- * This is to be used when writing a policy object in the federation policy
- * store.
- *
- * @return a valid policy configuration representing this object
- * parametrization.
- *
- * @throws FederationPolicyInitializationException if the current state cannot
- * be serialized properly
- */
- SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
new file mode 100644
index 0000000..a0fa37f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/WeightedPolicyInfo.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.dao;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This is a DAO class for the configuration of parameteres for federation
+ * policies. This generalizes several possible configurations as two lists of
+ * {@link SubClusterIdInfo} and corresponding weights as a
+ * {@link Float}. The interpretation of the weight is left to the logic in
+ * the policy.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@XmlRootElement(name = "federation-policy")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class WeightedPolicyInfo {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WeightedPolicyInfo.class);
+
+ private Map<SubClusterIdInfo, Float> routerPolicyWeights = new HashMap<>();
+ private Map<SubClusterIdInfo, Float> amrmPolicyWeights = new HashMap<>();
+ private float headroomAlpha;
+
+ private static JSONJAXBContext jsonjaxbContext = initContext();
+
+ private static JSONJAXBContext initContext() {
+ try {
+ return new JSONJAXBContext(JSONConfiguration.DEFAULT,
+ WeightedPolicyInfo.class);
+ } catch (JAXBException e) {
+ LOG.error("Error parsing the policy.", e);
+ }
+ return null;
+ }
+
+ public WeightedPolicyInfo() {
+ //JAXB needs this
+ }
+
+ /**
+ * Setter method for Router weights.
+ *
+ * @param policyWeights the router weights.
+ */
+ public void setRouterPolicyWeights(
+ Map<SubClusterIdInfo, Float> policyWeights) {
+ this.routerPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Setter method for ARMRMProxy weights.
+ *
+ * @param policyWeights the amrmproxy weights.
+ */
+ public void setAMRMPolicyWeights(
+ Map<SubClusterIdInfo, Float> policyWeights) {
+ this.amrmPolicyWeights = policyWeights;
+ }
+
+ /**
+ * Getter of the router weights.
+ * @return the router weights.
+ */
+ public Map<SubClusterIdInfo, Float> getRouterPolicyWeights() {
+ return routerPolicyWeights;
+ }
+
+ /**
+ * Getter for AMRMProxy weights.
+ * @return the AMRMProxy weights.
+ */
+ public Map<SubClusterIdInfo, Float> getAMRMPolicyWeights() {
+ return amrmPolicyWeights;
+ }
+
+ /**
+ * Deserializes a {@link WeightedPolicyInfo} from a byte UTF-8 JSON
+ * representation.
+ *
+ * @param bb the input byte representation.
+ *
+ * @return the {@link WeightedPolicyInfo} represented.
+ *
+ * @throws FederationPolicyInitializationException if a deserializaiton error
+ * occurs.
+ */
+ public static WeightedPolicyInfo fromByteBuffer(ByteBuffer bb)
+ throws FederationPolicyInitializationException {
+
+ if (jsonjaxbContext == null) {
+ throw new FederationPolicyInitializationException("JSONJAXBContext should"
+ + " not be null.");
+ }
+
+ try {
+ JSONUnmarshaller unmarshaller = jsonjaxbContext.createJSONUnmarshaller();
+ final byte[] bytes = new byte[bb.remaining()];
+ bb.get(bytes);
+ String params = new String(bytes, Charset.forName("UTF-8"));
+
+ WeightedPolicyInfo weightedPolicyInfo = unmarshaller
+ .unmarshalFromJSON(new StringReader(params),
+ WeightedPolicyInfo.class);
+ return weightedPolicyInfo;
+ } catch (JAXBException j) {
+ throw new FederationPolicyInitializationException(j);
+ }
+ }
+
+ /**
+ * Converts the policy into a byte array representation in the input {@link
+ * ByteBuffer}.
+ *
+ * @return byte array representation of this policy configuration.
+ *
+ * @throws FederationPolicyInitializationException if a serialization error
+ * occurs.
+ */
+ public ByteBuffer toByteBuffer()
+ throws FederationPolicyInitializationException {
+ if (jsonjaxbContext == null) {
+ throw new FederationPolicyInitializationException("JSONJAXBContext should"
+ + " not be null.");
+ }
+ try {
+ String s = toJSONString();
+ return ByteBuffer.wrap(s.getBytes(Charset.forName("UTF-8")));
+ } catch (JAXBException j) {
+ throw new FederationPolicyInitializationException(j);
+ }
+ }
+
+ private String toJSONString() throws JAXBException {
+ JSONMarshaller marshaller = jsonjaxbContext.createJSONMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+ StringWriter sw = new StringWriter(256);
+ marshaller.marshallToJSON(this, sw);
+ return sw.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+
+ if (other == null || !other.getClass().equals(this.getClass())) {
+ return false;
+ }
+
+ WeightedPolicyInfo otherPolicy =
+ (WeightedPolicyInfo) other;
+ Map<SubClusterIdInfo, Float> otherAMRMWeights =
+ otherPolicy.getAMRMPolicyWeights();
+ Map<SubClusterIdInfo, Float> otherRouterWeights =
+ otherPolicy.getRouterPolicyWeights();
+
+ boolean amrmWeightsMatch = otherAMRMWeights != null &&
+ getAMRMPolicyWeights() != null &&
+ CollectionUtils.isEqualCollection(otherAMRMWeights.entrySet(),
+ getAMRMPolicyWeights().entrySet());
+
+ boolean routerWeightsMatch = otherRouterWeights != null &&
+ getRouterPolicyWeights() != null &&
+ CollectionUtils.isEqualCollection(otherRouterWeights.entrySet(),
+ getRouterPolicyWeights().entrySet());
+
+ return amrmWeightsMatch && routerWeightsMatch;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * amrmPolicyWeights.hashCode() + routerPolicyWeights.hashCode();
+ }
+
+ /**
+ * Return the parameter headroomAlpha, used by policies that balance
+ * weight-based and load-based considerations in their decisions.
+ *
+ * For policies that use this parameter, values close to 1 indicate that
+ * most of the decision should be based on currently observed headroom from
+ * various sub-clusters, values close to zero, indicate that the decision
+ * should be mostly based on weights and practically ignore current load.
+ *
+ * @return the value of headroomAlpha.
+ */
+ public float getHeadroomAlpha() {
+ return headroomAlpha;
+ }
+
+ /**
+ * Set the parameter headroomAlpha, used by policies that balance
+ * weight-based and load-based considerations in their decisions.
+ *
+ * For policies that use this parameter, values close to 1 indicate that
+ * most of the decision should be based on currently observed headroom from
+ * various sub-clusters, values close to zero, indicate that the decision
+ * should be mostly based on weights and practically ignore current load.
+ *
+ * @param headroomAlpha the value to use for balancing.
+ */
+ public void setHeadroomAlpha(float headroomAlpha) {
+ this.headroomAlpha = headroomAlpha;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return toJSONString();
+ } catch (JAXBException e) {
+ e.printStackTrace();
+ return "Error serializing to string.";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
new file mode 100644
index 0000000..43f5b83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/dao/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** DAO objects for serializing/deserializing policy configurations. **/
+package org.apache.hadoop.yarn.server.federation.policies.dao;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
new file mode 100644
index 0000000..e888979
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseWeightedRouterPolicy.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.Map;
+
+/**
+ * Abstract class provides common validation of reinitialize(), for all
+ * policies that are "weight-based".
+ */
+public abstract class BaseWeightedRouterPolicy
+ implements FederationRouterPolicy {
+
+ private WeightedPolicyInfo policyInfo = null;
+ private FederationPolicyInitializationContext policyContext;
+
+ public BaseWeightedRouterPolicy() {
+ }
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext
+ federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ // perform consistency checks
+ WeightedPolicyInfo newPolicyInfo = WeightedPolicyInfo
+ .fromByteBuffer(
+ federationPolicyContext.getSubClusterPolicyConfiguration()
+ .getParams());
+
+ // if nothing has changed skip the rest of initialization
+ if (policyInfo != null && policyInfo.equals(newPolicyInfo)) {
+ return;
+ }
+
+ validate(newPolicyInfo);
+ setPolicyInfo(newPolicyInfo);
+ this.policyContext = federationPolicyContext;
+ }
+
+ /**
+ * Overridable validation step for the policy configuration.
+ * @param newPolicyInfo the configuration to test.
+ * @throws FederationPolicyInitializationException if the configuration is
+ * not valid.
+ */
+ public void validate(WeightedPolicyInfo newPolicyInfo) throws
+ FederationPolicyInitializationException {
+ if (newPolicyInfo == null) {
+ throw new FederationPolicyInitializationException("The policy to "
+ + "validate should not be null.");
+ }
+ Map<SubClusterIdInfo, Float> newWeights =
+ newPolicyInfo.getRouterPolicyWeights();
+ if (newWeights == null || newWeights.size() < 1) {
+ throw new FederationPolicyInitializationException(
+ "Weight vector cannot be null/empty.");
+ }
+ }
+
+
+ /**
+ * Getter method for the configuration weights.
+ *
+ * @return the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public WeightedPolicyInfo getPolicyInfo() {
+ return policyInfo;
+ }
+
+ /**
+ * Setter method for the configuration weights.
+ *
+ * @param policyInfo the {@link WeightedPolicyInfo} representing the policy
+ * configuration.
+ */
+ public void setPolicyInfo(
+ WeightedPolicyInfo policyInfo) {
+ this.policyInfo = policyInfo;
+ }
+
+ /**
+ * Getter method for the {@link FederationPolicyInitializationContext}.
+ * @return the context for this policy.
+ */
+ public FederationPolicyInitializationContext getPolicyContext() {
+ return policyContext;
+ }
+
+ /**
+ * Setter method for the {@link FederationPolicyInitializationContext}.
+ * @param policyContext the context to assign to this policy.
+ */
+ public void setPolicyContext(
+ FederationPolicyInitializationContext policyContext) {
+ this.policyContext = policyContext;
+ }
+
+ /**
+ * This methods gets active subclusters map from the {@code
+ * FederationStateStoreFacade} and validate it not being null/empty.
+ *
+ * @return the map of ids to info for all active subclusters.
+ * @throws YarnException if we can't get the list.
+ */
+ protected Map<SubClusterId, SubClusterInfo> getActiveSubclusters()
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters = getPolicyContext()
+ .getFederationStateStoreFacade().getSubClusters(true);
+
+ if (activeSubclusters == null || activeSubclusters.size() < 1) {
+ throw new NoActiveSubclustersException(
+ "Zero active subclusters, cannot pick where to send job.");
+ }
+ return activeSubclusters;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
new file mode 100644
index 0000000..e57709f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.Map;
+
+/**
+ * This implements a simple load-balancing policy. The policy "weights" are
+ * binary 0/1 values that enable/disable each sub-cluster, and the policy peaks
+ * the sub-cluster with the least load to forward this application.
+ */
+public class LoadBasedRouterPolicy
+ extends BaseWeightedRouterPolicy {
+
+ private static final Log LOG =
+ LogFactory.getLog(LoadBasedRouterPolicy.class);
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext
+ federationPolicyContext)
+ throws FederationPolicyInitializationException {
+
+ // remember old policyInfo
+ WeightedPolicyInfo tempPolicy = getPolicyInfo();
+
+ //attempt new initialization
+ super.reinitialize(federationPolicyContext);
+
+ //check extra constraints
+ for (Float weight : getPolicyInfo().getRouterPolicyWeights().values()) {
+ if (weight != 0 && weight != 1) {
+ //reset to old policyInfo if check fails
+ setPolicyInfo(tempPolicy);
+ throw new FederationPolicyInitializationException(
+ this.getClass().getCanonicalName()
+ + " policy expects all weights to be either "
+ + "\"0\" or \"1\"");
+ }
+ }
+ }
+
+ @Override
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
+ .getRouterPolicyWeights();
+ SubClusterIdInfo chosen = null;
+ long currBestMem = -1;
+ for (Map.Entry<SubClusterId, SubClusterInfo> entry :
+ activeSubclusters
+ .entrySet()) {
+ SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey());
+ if (weights.containsKey(id) && weights.get(id) > 0) {
+ long availableMemory = getAvailableMemory(entry.getValue());
+ if (availableMemory > currBestMem) {
+ currBestMem = availableMemory;
+ chosen = id;
+ }
+ }
+ }
+
+ return chosen.toId();
+ }
+
+ private long getAvailableMemory(SubClusterInfo value)
+ throws YarnException {
+ try {
+ long mem = -1;
+ JSONObject obj = new JSONObject(value.getCapability());
+ mem = obj.getJSONObject("clusterMetrics").getLong("availableMB");
+ return mem;
+ } catch (JSONException j) {
+ throw new YarnException("FederationSubCluserInfo cannot be parsed", j);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
new file mode 100644
index 0000000..a8ac5f7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.Map;
+
+/**
+ * This implements a policy that interprets "weights" as a ordered list of
+ * preferences among sub-clusters. Highest weight among active subclusters is
+ * chosen.
+ */
+public class PriorityRouterPolicy
+ extends BaseWeightedRouterPolicy {
+
+ private static final Log LOG =
+ LogFactory.getLog(PriorityRouterPolicy.class);
+
+ @Override
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ // This finds the sub-cluster with the highest weight among the
+ // currently active ones.
+ Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
+ .getRouterPolicyWeights();
+ SubClusterId chosen = null;
+ Float currentBest = Float.MIN_VALUE;
+ for (SubClusterId id : activeSubclusters.keySet()) {
+ SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
+ if (weights.containsKey(idInfo) && weights.get(idInfo) > currentBest) {
+ currentBest = weights.get(idInfo);
+ chosen = id;
+ }
+ }
+
+ return chosen;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
new file mode 100644
index 0000000..1774961
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * This simple policy picks at uniform random among any of the currently active
+ * subclusters. This policy is easy to use and good for testing.
+ *
+ * NOTE: this is "almost" subsumed by the {@code WeightedRandomRouterPolicy}.
+ * Behavior only diverges when there are active sub-clusters that are not part
+ * of the "weights", in which case the {@link UniformRandomRouterPolicy} send
+ * load to them, while {@code WeightedRandomRouterPolicy} does not.
+ */
+public class UniformRandomRouterPolicy extends BaseWeightedRouterPolicy {
+
+ private Random rand;
+
+ public UniformRandomRouterPolicy() {
+ rand = new Random(System.currentTimeMillis());
+ }
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ //note: this overrides BaseWeighterRouterPolicy and ignores the weights
+
+ setPolicyContext(federationPolicyContext);
+ }
+
+ /**
+ * Simply picks a random active subcluster to start the AM (this does NOT
+ * depend on the weights in the policy).
+ *
+ * @param appSubmissionContext the context for the app being submitted
+ * (ignored).
+ *
+ * @return a randomly chosen subcluster.
+ *
+ * @throws YarnException if there are no active subclusters.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ List<SubClusterId> list =
+ new ArrayList<>(activeSubclusters.keySet());
+ return list.get(rand.nextInt(list.size()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
new file mode 100644
index 0000000..0777677
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * This policy implements a weighted random sample among currently active
+ * sub-clusters.
+ */
+public class WeightedRandomRouterPolicy
+ extends BaseWeightedRouterPolicy {
+
+ private static final Log LOG =
+ LogFactory.getLog(WeightedRandomRouterPolicy.class);
+ private Random rand = new Random(System.currentTimeMillis());
+
+ @Override
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException {
+
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ // note: we cannot pre-compute the weights, as the set of activeSubcluster
+ // changes dynamically (and this would unfairly spread the load to
+ // sub-clusters adjacent to an inactive one), hence we need to count/scan
+ // the list and based on weight pick the next sub-cluster.
+ Map<SubClusterIdInfo, Float> weights = getPolicyInfo()
+ .getRouterPolicyWeights();
+
+ float totActiveWeight = 0;
+ for(Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()){
+ if(entry.getKey()!=null && activeSubclusters.containsKey(entry.getKey()
+ .toId())){
+ totActiveWeight += entry.getValue();
+ }
+ }
+ float lookupValue = rand.nextFloat() * totActiveWeight;
+
+ for (SubClusterId id : activeSubclusters.keySet()) {
+ SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
+ if (weights.containsKey(idInfo)) {
+ lookupValue -= weights.get(idInfo);
+ }
+ if (lookupValue <= 0) {
+ return id;
+ }
+ }
+ //should never happen
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
new file mode 100644
index 0000000..e2260a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class represent a sub-cluster identifier in the JSON representation
+ * of the policy configuration.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@XmlRootElement(name = "federation-policy")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SubClusterIdInfo {
+
+ private String id;
+
+ public SubClusterIdInfo() {
+ //JAXB needs this
+ }
+
+ public SubClusterIdInfo(String subClusterId) {
+ this.id = subClusterId;
+ }
+
+ public SubClusterIdInfo(SubClusterId subClusterId) {
+ this.id = subClusterId.getId();
+ }
+
+ /**
+ * Get the sub-cluster identifier as {@link SubClusterId}.
+ * @return the sub-cluster id.
+ */
+ public SubClusterId toId() {
+ return SubClusterId.newInstance(id);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof SubClusterIdInfo) {
+ if (((SubClusterIdInfo) other).id.equals(this.id)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
new file mode 100644
index 0000000..8da92b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import static org.mockito.Mockito.mock;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Test;
+
+/**
+ * Base class for policies tests, tests for common reinitialization cases.
+ */
+public abstract class BaseFederationPoliciesTest {
+
+ private ConfigurableFederationPolicy policy;
+ private WeightedPolicyInfo policyInfo;
+ private Map<SubClusterId, SubClusterInfo> activeSubclusters = new HashMap<>();
+ private FederationPolicyInitializationContext federationPolicyContext;
+ private ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ private Random rand = new Random();
+
+ @Test
+ public void testReinitilialize() throws YarnException {
+ FederationPolicyInitializationContext fpc =
+ new FederationPolicyInitializationContext();
+ ByteBuffer buf = getPolicyInfo().toByteBuffer();
+ fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+ .newInstance("queue1", getPolicy().getClass().getCanonicalName(), buf));
+ fpc.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ getPolicy().reinitialize(fpc);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testReinitilializeBad1() throws YarnException {
+ getPolicy().reinitialize(null);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testReinitilializeBad2() throws YarnException {
+ FederationPolicyInitializationContext fpc =
+ new FederationPolicyInitializationContext();
+ getPolicy().reinitialize(fpc);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testReinitilializeBad3() throws YarnException {
+ FederationPolicyInitializationContext fpc =
+ new FederationPolicyInitializationContext();
+ ByteBuffer buf = mock(ByteBuffer.class);
+ fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+ .newInstance("queue1", "WrongPolicyName", buf));
+ fpc.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ fpc.setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ getPolicy().reinitialize(fpc);
+ }
+
+ @Test(expected = NoActiveSubclustersException.class)
+ public void testNoSubclusters() throws YarnException {
+ // empty the activeSubclusters map
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), new HashMap<>());
+
+ ConfigurableFederationPolicy currentPolicy = getPolicy();
+ if (currentPolicy instanceof FederationRouterPolicy) {
+ ((FederationRouterPolicy) currentPolicy)
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ }
+ }
+
+ public ConfigurableFederationPolicy getPolicy() {
+ return policy;
+ }
+
+ public void setPolicy(ConfigurableFederationPolicy policy) {
+ this.policy = policy;
+ }
+
+ public WeightedPolicyInfo getPolicyInfo() {
+ return policyInfo;
+ }
+
+ public void setPolicyInfo(WeightedPolicyInfo policyInfo) {
+ this.policyInfo = policyInfo;
+ }
+
+ public Map<SubClusterId, SubClusterInfo> getActiveSubclusters() {
+ return activeSubclusters;
+ }
+
+ public void setActiveSubclusters(
+ Map<SubClusterId, SubClusterInfo> activeSubclusters) {
+ this.activeSubclusters = activeSubclusters;
+ }
+
+ public FederationPolicyInitializationContext getFederationPolicyContext() {
+ return federationPolicyContext;
+ }
+
+ public void setFederationPolicyContext(
+ FederationPolicyInitializationContext federationPolicyContext) {
+ this.federationPolicyContext = federationPolicyContext;
+ }
+
+ public ApplicationSubmissionContext getApplicationSubmissionContext() {
+ return applicationSubmissionContext;
+ }
+
+ public void setApplicationSubmissionContext(
+ ApplicationSubmissionContext applicationSubmissionContext) {
+ this.applicationSubmissionContext = applicationSubmissionContext;
+ }
+
+ public Random getRand() {
+ return rand;
+ }
+
+ public void setRand(Random rand) {
+ this.rand = rand;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index 4ec04d5..e840b3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -77,7 +77,7 @@ public class TestFederationPolicyInitializationContextValidator {
@Test(expected = FederationPolicyInitializationException.class)
public void nullConf() throws Exception {
- context.setFederationPolicyConfiguration(null);
+ context.setSubClusterPolicyConfiguration(null);
FederationPolicyInitializationContextValidator.validate(context,
MockPolicyManager.class.getCanonicalName());
}
@@ -96,8 +96,8 @@ public class TestFederationPolicyInitializationContextValidator {
MockPolicyManager.class.getCanonicalName());
}
- private class MockPolicyManager
- implements FederationPolicyWriter, FederationPolicyConfigurator {
+ private class MockPolicyManager implements FederationPolicyManager {
+
@Override
public FederationAMRMProxyPolicy getAMRMPolicy(
FederationPolicyInitializationContext
@@ -123,6 +123,17 @@ public class TestFederationPolicyInitializationContextValidator {
return SubClusterPolicyConfiguration
.newInstance("queue1", this.getClass().getCanonicalName(), buf);
}
+
+ @Override
+ public String getQueue() {
+ return "default";
+ }
+
+ @Override
+ public void setQueue(String queue) {
+
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
new file mode 100644
index 0000000..9e94f72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the
+ * load is properly considered for allocation.
+ */
+public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new LoadBasedRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 active subclusters
+ for (int i = 0; i < 20; i++) {
+ SubClusterIdInfo sc =
+ new SubClusterIdInfo(String.format("sc%02d", i));
+ SubClusterInfo federationSubClusterInfo =
+ SubClusterInfo.newInstance(sc.toId(), null, null, null, null, -1,
+ SubClusterState.SC_RUNNING, -1,
+ generateClusterMetricsInfo(i));
+ getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
+ float weight = getRand().nextInt(2);
+ if (i == 5) {
+ weight = 1.0f;
+ }
+
+ // 5% chance we omit one of the weights
+ if (i <= 5 || getRand().nextFloat() > 0.05f) {
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ }
+ }
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+
+ }
+
+ private String generateClusterMetricsInfo(int id) {
+
+ long mem = 1024 * getRand().nextInt(277 * 100 - 1);
+ //plant a best cluster
+ if (id == 5) {
+ mem = 1024 * 277 * 100;
+ }
+ String clusterMetrics =
+ "{\"clusterMetrics\":{\"appsSubmitted\":65," + "\"appsCompleted\":64,"
+ + "\"appsPending\":0,\"appsRunning\":0,\"appsFailed\":0,"
+ + "\"appsKilled\":1,\"reservedMB\":0,\"availableMB\":" + mem + ","
+ + "\"allocatedMB\":0,\"reservedVirtualCores\":0,"
+ + "\"availableVirtualCores\":2216,\"allocatedVirtualCores\":0,"
+ + "\"containersAllocated\":0,\"containersReserved\":0,"
+ + "\"containersPending\":0,\"totalMB\":28364800,"
+ + "\"totalVirtualCores\":2216,\"totalNodes\":278,\"lostNodes\":1,"
+ + "\"unhealthyNodes\":0,\"decommissionedNodes\":0,"
+ + "\"rebootedNodes\":0,\"activeNodes\":277}}\n";
+
+ return clusterMetrics;
+
+ }
+
+ @Test
+ public void testLoadIsRespected() throws YarnException {
+
+ SubClusterId chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+
+ // check the "planted" best cluster is chosen
+ Assert.assertEquals("sc05", chosen.getId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
new file mode 100644
index 0000000..ff5175d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Simple test class for the {@link PriorityRouterPolicy}. Tests that the
+ * weights are correctly used for ordering the choice of sub-clusters.
+ */
+public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new PriorityRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 subclusters with a 5% chance of being inactive
+ for (int i = 0; i < 20; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+
+ // with 5% omit a subcluster
+ if (getRand().nextFloat() < 0.95f || i == 5) {
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+ float weight = getRand().nextFloat();
+ if (i == 5) {
+ weight = 1.1f; // guaranteed to be the largest.
+ }
+
+ // 5% chance we omit one of the weights
+ if (i <= 5 || getRand().nextFloat() > 0.05f) {
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ }
+ }
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(),
+ getActiveSubclusters());
+
+ }
+
+ @Test
+ public void testPickLowestWeight() throws YarnException {
+ SubClusterId chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ Assert.assertEquals("sc5", chosen.getId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
new file mode 100644
index 0000000..ac41ab5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link UniformRandomRouterPolicy}. Tests that one
+ * of the active subcluster is chosen.
+ */
+public class TestUniformRandomRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new UniformRandomRouterPolicy());
+ // needed for base test to work
+ setPolicyInfo(mock(WeightedPolicyInfo.class));
+ for (int i = 1; i <= 2; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ mock(WeightedPolicyInfo.class), getActiveSubclusters());
+ }
+
+ @Test
+ public void testOneSubclusterIsChosen() throws YarnException {
+ SubClusterId chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ Assert.assertTrue(getActiveSubclusters().keySet().contains(chosen));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
new file mode 100644
index 0000000..a612685
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Simple test class for the {@link WeightedRandomRouterPolicy}. Generate large
+ * number of randomized tests to check we are weighiting correctly even if
+ * clusters go inactive.
+ */
+public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new WeightedRandomRouterPolicy());
+ setPolicyInfo(new WeightedPolicyInfo());
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+
+ // simulate 20 subclusters with a 5% chance of being inactive
+ for (int i = 0; i < 20; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ // with 5% omit a subcluster
+ if (getRand().nextFloat() < 0.95f) {
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+ // 5% chance we omit one of the weights
+ float weight = getRand().nextFloat();
+ if (i <= 5 || getRand().nextFloat() > 0.05f) {
+ routerWeights.put(sc, weight);
+ amrmWeights.put(sc, weight);
+ }
+ }
+ getPolicyInfo().setRouterPolicyWeights(routerWeights);
+ getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(),
+ getActiveSubclusters());
+
+ }
+
+ @Test
+ public void testClusterChosenWithRightProbability() throws YarnException {
+
+ Map<SubClusterId, AtomicLong> counter = new HashMap<>();
+ for (SubClusterIdInfo id : getPolicyInfo().getRouterPolicyWeights()
+ .keySet()) {
+ counter.put(id.toId(), new AtomicLong(0));
+ }
+
+ float numberOfDraws = 1000000;
+
+ for (float i = 0; i < numberOfDraws; i++) {
+ SubClusterId chosenId = ((FederationRouterPolicy) getPolicy()).
+ getHomeSubcluster(getApplicationSubmissionContext());
+ counter.get(chosenId).incrementAndGet();
+ }
+
+ float totalActiveWeight = 0;
+ for (SubClusterId id : getActiveSubclusters().keySet()) {
+ SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
+ if (getPolicyInfo().getRouterPolicyWeights().containsKey(idInfo)) {
+ totalActiveWeight +=
+ getPolicyInfo().getRouterPolicyWeights().get(idInfo);
+ }
+ }
+
+ for (Map.Entry<SubClusterId, AtomicLong> counterEntry : counter
+ .entrySet()) {
+ float expectedWeight = getPolicyInfo().getRouterPolicyWeights()
+ .get(new SubClusterIdInfo(counterEntry.getKey())) / totalActiveWeight;
+ float actualWeight = counterEntry.getValue().floatValue() / numberOfDraws;
+
+ // make sure that the weights is respected among active subclusters
+ // and no jobs are routed to inactive subclusters.
+ if (getActiveSubclusters().containsKey(counterEntry.getKey())) {
+ Assert.assertTrue(
+ "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ + " expected weight: " + expectedWeight, expectedWeight == 0 ||
+ (actualWeight / expectedWeight) < 1.1
+ && (actualWeight / expectedWeight) > 0.9);
+ } else {
+ Assert.assertTrue(
+ "Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ + " expected weight: " + expectedWeight, actualWeight == 0);
+
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDFS-11427. Rename rs-default to
rs.
Posted by su...@apache.org.
HDFS-11427. Rename rs-default to rs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1a52b04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1a52b04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1a52b04
Branch: refs/heads/YARN-2915
Commit: c1a52b04d0cc5ad5c86ae93043655f313386f7f9
Parents: b32ffa2
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Feb 24 10:58:45 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Feb 24 10:58:45 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/io/erasurecode/CodecUtil.java | 28 +++++++--------
.../io/erasurecode/ErasureCodeConstants.java | 8 ++---
.../erasurecode/coder/HHXORErasureDecoder.java | 2 +-
.../erasurecode/coder/HHXORErasureEncoder.java | 2 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 2 +-
.../io/erasurecode/coder/RSErasureEncoder.java | 2 +-
.../src/main/resources/core-default.xml | 4 +--
.../erasurecode/TestCodecRawCoderMapping.java | 10 +++---
.../coder/TestHHXORErasureCoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 2 +-
.../src/site/markdown/HDFSErasureCoding.md | 6 ++--
.../TestDFSRSDefault10x4StripedInputStream.java | 2 +-
...TestDFSRSDefault10x4StripedOutputStream.java | 2 +-
...fault10x4StripedOutputStreamWithFailure.java | 4 +--
.../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +-
.../TestDFSStripedOutputStreamWithFailure.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 36 ++++++++++----------
20 files changed, 61 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 977bacb..861451a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -55,9 +55,9 @@ public final class CodecUtil {
public static final String IO_ERASURECODE_CODEC_XOR =
XORErasureCodec.class.getCanonicalName();
/** Erasure coder Reed-Solomon codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY =
+ public static final String IO_ERASURECODE_CODEC_RS_KEY =
"io.erasurecode.codec.rs";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT =
+ public static final String IO_ERASURECODE_CODEC_RS =
RSErasureCodec.class.getCanonicalName();
/** Erasure coder hitch hiker XOR codec. */
public static final String IO_ERASURECODE_CODEC_HHXOR_KEY =
@@ -67,10 +67,10 @@ public final class CodecUtil {
/** Supported erasure codec classes. */
- /** Raw coder factory for the RS default codec. */
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
- "io.erasurecode.codec.rs-default.rawcoder";
- public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
+ /** Raw coder factory for the RS codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ "io.erasurecode.codec.rs.rawcoder";
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT =
RSRawErasureCoderFactory.class.getCanonicalName();
/** Raw coder factory for the RS legacy codec. */
@@ -183,10 +183,10 @@ public final class CodecUtil {
private static String getRawCoderFactNameFromCodec(Configuration conf,
String codec) {
switch (codec) {
- case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ case ErasureCodeConstants.RS_CODEC_NAME:
return conf.get(
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
- IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
+ IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
return conf.get(
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
@@ -233,15 +233,15 @@ public final class CodecUtil {
private static String getCodecClassName(Configuration conf, String codec) {
switch (codec) {
- case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
+ case ErasureCodeConstants.RS_CODEC_NAME:
return conf.get(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ CodecUtil.IO_ERASURECODE_CODEC_RS_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
//TODO:rs-legacy should be handled differently.
return conf.get(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY,
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT);
+ CodecUtil.IO_ERASURECODE_CODEC_RS_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS);
case ErasureCodeConstants.XOR_CODEC_NAME:
return conf.get(
CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index e168909..c830bb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -25,16 +25,16 @@ public final class ErasureCodeConstants {
private ErasureCodeConstants() {
}
- public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
+ public static final String RS_CODEC_NAME = "rs";
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final String HHXOR_CODEC_NAME = "hhxor";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 6, 3);
+ RS_CODEC_NAME, 6, 3);
public static final ECSchema RS_3_2_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 3, 2);
+ RS_CODEC_NAME, 3, 2);
public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
RS_LEGACY_CODEC_NAME, 6, 3);
@@ -43,5 +43,5 @@ public final class ErasureCodeConstants {
XOR_CODEC_NAME, 2, 1);
public static final ECSchema RS_10_4_SCHEMA = new ECSchema(
- RS_DEFAULT_CODEC_NAME, 10, 4);
+ RS_CODEC_NAME, 10, 4);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
index 05e9384..4747e49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java
@@ -67,7 +67,7 @@ public class HHXORErasureDecoder extends ErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
index 7a15a05..6b858b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
@@ -61,7 +61,7 @@ public class HHXORErasureEncoder extends ErasureEncoder {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
rsRawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
index 6e679c3..ba7f773 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -52,7 +52,7 @@ public class RSErasureDecoder extends ErasureDecoder {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
rsRawDecoder = CodecUtil.createRawDecoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rsRawDecoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
index 7a09b92..193309d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -53,7 +53,7 @@ public class RSErasureEncoder extends ErasureEncoder {
if (rawEncoder == null) {
// TODO: we should create the raw coder according to codec.
rawEncoder = CodecUtil.createRawEncoder(getConf(),
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions());
+ ErasureCodeConstants.RS_CODEC_NAME, getOptions());
}
return rawEncoder;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index d8136ee..08ca05b 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -657,10 +657,10 @@
</property>
<property>
- <name>io.erasurecode.codec.rs-default.rawcoder</name>
+ <name>io.erasurecode.codec.rs.rawcoder</name>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
<description>
- Raw coder implementation for the rs-default codec. The default value is a
+ Raw coder implementation for the rs codec. The default value is a
pure Java implementation. There is also a native implementation. Its value
is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
index 0db001a..7f7fcf3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -47,12 +47,12 @@ public class TestCodecRawCoderMapping {
public void testRSDefaultRawCoder() {
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
numDataUnit, numParityUnit);
- // should return default raw coder of rs-default codec
+ // should return default raw coder of rs codec
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
- conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
RawErasureDecoder decoder = CodecUtil.createRawDecoder(
- conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(decoder instanceof RSRawDecoder);
// should return default raw coder of rs-legacy codec
@@ -71,11 +71,11 @@ public class TestCodecRawCoderMapping {
String dummyFactName = "DummyNoneExistingFactory";
// set the dummy factory to rs-legacy and create a raw coder
- // with rs-default, which is OK as the raw coder key is not used
+ // with rs, which is OK as the raw coder key is not used
conf.set(CodecUtil.
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf,
- ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions);
+ ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
// now create the raw coder with rs-legacy, which should throw exception
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index 1eca1e1..91c4ba8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index 5f36c99..3b18347 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
- conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 0283e2b..36fb61d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -64,7 +64,7 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
- There are four policies currently being supported: RS-DEFAULT-3-2-64k, RS-DEFAULT-6-3-64k, RS-DEFAULT-10-4-64k and RS-LEGACY-6-3-64k. All with default cell size of 64KB. The system default policy is RS-DEFAULT-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB.
+ Five policies are currently supported: RS-3-2-64k, RS-6-3-64k, RS-10-4-64k, RS-LEGACY-6-3-64k, and XOR-2-1-64k. All with default cell size of 64KB. The system default policy is RS-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB.
* **Intel ISA-L**
Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets.
@@ -91,7 +91,7 @@ Deployment
### Configuration keys
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
- `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec,
+ `io.erasurecode.codec.rs.rawcoder` for the default RS codec,
`io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
`io.erasurecode.codec.xor.rawcoder` for the XOR codec.
The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
@@ -108,7 +108,7 @@ Deployment
HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps.
1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/" for detail information.
2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes.
- 3. Configure the `io.erasurecode.codec.rs-default.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes.
+ 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes.
To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
index fc0ee37..3e6d1e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests read operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests read operation of DFS striped file with RS-10-4-64k
* erasure code policy.
*/
public class TestDFSRSDefault10x4StripedInputStream extends
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
index 37821c1..1ea839a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests write operation of DFS striped file with RS-10-4-64k
* erasure code policy.
*/
public class TestDFSRSDefault10x4StripedOutputStream
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
index 1b2ec42..340fec5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
/**
- * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k
+ * This tests write operation of DFS striped file with RS-10-4-64k
* erasure code policy under Datanode failure conditions.
*/
public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
@@ -33,4 +33,4 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
return ErasureCodingPolicyManager.getPolicyByPolicyID(
HdfsConstants.RS_10_4_POLICY_ID);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 121b9a4..9b69904 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -96,7 +96,7 @@ public class TestDFSStripedInputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
SimulatedFSDataset.setFactory(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 5bde16e..8834e14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 0baf9cc..fabd417 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
final int numDNs = dataBlocks + parityBlocks;
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 4960c58..08b53c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -100,7 +100,7 @@ public class TestReconstructStripedFile {
false);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index 1a4086e..f26aa06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -69,7 +69,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set(
- CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
NativeRSRawErasureCoderFactory.class.getCanonicalName());
}
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a52b04/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 82b71def..9b9003a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -133,7 +133,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -141,7 +141,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -150,8 +150,8 @@
<description>setPolicy : set a policy twice</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -159,7 +159,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir</expected-output>
+ <expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output>
</comparator>
</comparators>
</test>
@@ -168,7 +168,7 @@
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
@@ -187,8 +187,8 @@
<description>setPolicy : change different policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -197,7 +197,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-3-2-64k</expected-output>
+ <expected-output>RS-3-2-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -207,7 +207,7 @@
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -mkdir /ecdir/child</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command>
@@ -220,7 +220,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -246,7 +246,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@@ -255,7 +255,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -264,7 +264,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command>
</test-commands>
@@ -275,7 +275,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3-64k</expected-output>
+ <expected-output>RS-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@@ -290,7 +290,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>RS-DEFAULT-6-3</expected-output>
+ <expected-output>RS-6-3</expected-output>
</comparator>
</comparators>
</test>
@@ -334,7 +334,7 @@
<description>setPolicy : illegal parameters - too many arguments</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
- <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@@ -367,7 +367,7 @@
<test>
<description>setPolicy : illegal parameters - no such file</description>
<test-commands>
- <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k</ec-admin-command>
</test-commands>
<cleanup-commands>
</cleanup-commands>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-14097. Remove Java6
specific code from GzipCodec.java. Contributed by Elek, Marton.
Posted by su...@apache.org.
HADOOP-14097. Remove Java6 specific code from GzipCodec.java. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50decd36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50decd36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50decd36
Branch: refs/heads/YARN-2915
Commit: 50decd36130945e184734dcd55b8912be6f4550a
Parents: e60c654
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Feb 25 00:28:31 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Sat Feb 25 00:28:31 2017 +0900
----------------------------------------------------------------------
.../apache/hadoop/io/compress/GzipCodec.java | 59 --------------------
1 file changed, 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50decd36/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
index 01b6434..d079412 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
@@ -45,10 +45,6 @@ public class GzipCodec extends DefaultCodec {
protected static class GzipOutputStream extends CompressorStream {
private static class ResetableGZIPOutputStream extends GZIPOutputStream {
- private static final int TRAILER_SIZE = 8;
- public static final String JVMVersion= System.getProperty("java.version");
- private static final boolean HAS_BROKEN_FINISH =
- (IBM_JAVA && JVMVersion.contains("1.6.0"));
public ResetableGZIPOutputStream(OutputStream out) throws IOException {
super(out);
@@ -57,61 +53,6 @@ public class GzipCodec extends DefaultCodec {
public void resetState() throws IOException {
def.reset();
}
-
- /**
- * Override this method for HADOOP-8419.
- * Override because IBM implementation calls def.end() which
- * causes problem when reseting the stream for reuse.
- *
- */
- @Override
- public void finish() throws IOException {
- if (HAS_BROKEN_FINISH) {
- if (!def.finished()) {
- def.finish();
- while (!def.finished()) {
- int i = def.deflate(this.buf, 0, this.buf.length);
- if ((def.finished()) && (i <= this.buf.length - TRAILER_SIZE)) {
- writeTrailer(this.buf, i);
- i += TRAILER_SIZE;
- out.write(this.buf, 0, i);
-
- return;
- }
- if (i > 0) {
- out.write(this.buf, 0, i);
- }
- }
-
- byte[] arrayOfByte = new byte[TRAILER_SIZE];
- writeTrailer(arrayOfByte, 0);
- out.write(arrayOfByte);
- }
- } else {
- super.finish();
- }
- }
-
- /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
- private void writeTrailer(byte[] paramArrayOfByte, int paramInt)
- throws IOException {
- writeInt((int)this.crc.getValue(), paramArrayOfByte, paramInt);
- writeInt(this.def.getTotalIn(), paramArrayOfByte, paramInt + 4);
- }
-
- /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
- private void writeInt(int paramInt1, byte[] paramArrayOfByte, int paramInt2)
- throws IOException {
- writeShort(paramInt1 & 0xFFFF, paramArrayOfByte, paramInt2);
- writeShort(paramInt1 >> 16 & 0xFFFF, paramArrayOfByte, paramInt2 + 2);
- }
-
- /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
- private void writeShort(int paramInt1, byte[] paramArrayOfByte, int paramInt2)
- throws IOException {
- paramArrayOfByte[paramInt2] = (byte)(paramInt1 & 0xFF);
- paramArrayOfByte[(paramInt2 + 1)] = (byte)(paramInt1 >> 8 & 0xFF);
- }
}
public GzipOutputStream(OutputStream out) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-14102. Relax error message
assertion in S3A test ITestS3AEncryptionSSEC. Contributed by Mingliang Liu
Posted by su...@apache.org.
HADOOP-14102. Relax error message assertion in S3A test ITestS3AEncryptionSSEC. Contributed by Mingliang Liu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0013090f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0013090f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0013090f
Branch: refs/heads/YARN-2915
Commit: 0013090fb4340eadf147054e65a73de20a62c1c1
Parents: 1a6ca75
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Feb 21 17:30:39 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Feb 22 13:34:20 2017 -0800
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0013090f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
index 71586b8..a8cf70b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
@@ -58,7 +58,7 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
Exception {
final Path[] path = new Path[1];
intercept(java.nio.file.AccessDeniedException.class,
- "Forbidden (Service: Amazon S3; Status Code: 403;", () -> {
+ "Service: Amazon S3; Status Code: 403;", () -> {
int len = 2048;
skipIfEncryptionTestsDisabled(getConfiguration());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-5408. Compose Federation
membership/application/policy APIs into an uber FederationStateStore API.
(Ellen Hui via Subru).
Posted by su...@apache.org.
YARN-5408. Compose Federation membership/application/policy APIs into an uber FederationStateStore API. (Ellen Hui via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca9c2236
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca9c2236
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca9c2236
Branch: refs/heads/YARN-2915
Commit: ca9c223655dfa3f90f75c435f0f54e1ee81a27c6
Parents: 54f9a01
Author: Subru Krishnan <su...@apache.org>
Authored: Mon Aug 8 14:53:38 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
...ederationApplicationHomeSubClusterStore.java | 18 ++----
.../store/FederationMembershipStateStore.java | 14 +----
.../federation/store/FederationStateStore.java | 64 ++++++++++++++++++++
.../store/impl/MemoryFederationStateStore.java | 19 ------
.../impl/FederationStateStoreBaseTest.java | 57 +++++++++--------
.../impl/TestMemoryFederationStateStore.java | 21 +------
6 files changed, 99 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca9c2236/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
index 217ee2e..22bb88a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHom
import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
/**
* FederationApplicationHomeSubClusterStore maintains the state of all
@@ -50,15 +49,6 @@ import org.apache.hadoop.yarn.server.records.Version;
public interface FederationApplicationHomeSubClusterStore {
/**
- * Get the {@link Version} of the underlying federation application state
- * store.
- *
- * @return the {@link Version} of the underlying federation application state
- * store
- */
- Version getApplicationStateStoreVersion();
-
- /**
* Register the home {@code SubClusterId} of the newly submitted
* {@code ApplicationId}. Currently response is empty if the operation was
* successful, if not an exception reporting reason for a failure.
@@ -91,16 +81,16 @@ public interface FederationApplicationHomeSubClusterStore {
* {@code ApplicationId}.
*
* @param request contains the application queried
- * @return {@code ApplicationHomeSubCluster} containing the application's
- * home subcluster
+ * @return {@code ApplicationHomeSubCluster} containing the application's home
+ * subcluster
* @throws YarnException if the request is invalid/fails
*/
GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
GetApplicationHomeSubClusterRequest request) throws YarnException;
/**
- * Get the {@code ApplicationHomeSubCluster} list representing the mapping
- * of all submitted applications to it's home sub-cluster.
+ * Get the {@code ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
*
* @param request empty representing all applications
* @return the mapping of all submitted application to it's home sub-cluster
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca9c2236/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
index 378eadc..7778d5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbea
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
/**
* FederationMembershipStateStore maintains the state of all
@@ -42,15 +41,6 @@ import org.apache.hadoop.yarn.server.records.Version;
public interface FederationMembershipStateStore {
/**
- * Get the {@link Version} of the underlying federation membership state
- * store.
- *
- * @return the {@link Version} of the underlying federation membership state
- * store
- */
- Version getMembershipStateStoreVersion();
-
- /**
* Register a <em>subcluster</em> by publishing capabilities as represented by
* {@code SubClusterInfo} to indicate participation in federation. This is
* typically done during initialization or restart/failover of the
@@ -80,7 +70,7 @@ public interface FederationMembershipStateStore {
*/
SubClusterDeregisterResponse deregisterSubCluster(
SubClusterDeregisterRequest subClusterDeregisterRequest)
- throws YarnException;
+ throws YarnException;
/**
* Periodic heartbeat from a <code>ResourceManager</code> participating in
@@ -96,7 +86,7 @@ public interface FederationMembershipStateStore {
*/
SubClusterHeartbeatResponse subClusterHeartbeat(
SubClusterHeartbeatRequest subClusterHeartbeatRequest)
- throws YarnException;
+ throws YarnException;
/**
* Get the membership information of <em>subcluster</em> as identified by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca9c2236/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
new file mode 100644
index 0000000..9397e9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.records.Version;
+
+/**
+ * FederationStore extends the three interfaces used to coordinate the state of
+ * a federated cluster: {@link FederationApplicationHomeSubClusterStore},
+ * {@link FederationMembershipStateStore}, and {@link FederationPolicyStore}.
+ *
+ */
+public interface FederationStateStore
+ extends FederationApplicationHomeSubClusterStore,
+ FederationMembershipStateStore, FederationPolicyStore {
+
+ /**
+ * Initialize the FederationStore.
+ *
+ * @param conf the cluster configuration
+ * @throws YarnException if initialization fails
+ */
+ void init(Configuration conf) throws YarnException;
+
+ /**
+ * Perform any cleanup operations of the StateStore.
+ *
+ * @throws Exception if cleanup fails
+ */
+ void close() throws Exception;
+
+ /**
+ * Get the {@link Version} of the underlying federation state store client.
+ *
+ * @return the {@link Version} of the underlying federation store client
+ */
+ Version getCurrentVersion();
+
+ /**
+ * Load the version information from the federation state store.
+ *
+ * @return the {@link Version} of the federation state store
+ */
+ Version loadVersion();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca9c2236/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 7fdc4a9..cea4ac2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -36,11 +36,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbea
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* In-memory implementation of FederationMembershipStateStore.
*/
@@ -52,11 +49,6 @@ public class MemoryFederationStateStore
private final MonotonicClock clock = new MonotonicClock();
@Override
- public Version getMembershipStateStoreVersion() {
- return null;
- }
-
- @Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
@@ -122,17 +114,6 @@ public class MemoryFederationStateStore
}
}
return GetSubClustersInfoResponse.newInstance(result);
-
- }
-
- @VisibleForTesting
- public Map<SubClusterId, SubClusterInfo> getMembershipTable() {
- return membership;
- }
-
- @VisibleForTesting
- public void clearMembershipTable() {
- membership.clear();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca9c2236/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 7eb1c86..c76a485 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -19,25 +19,21 @@ package org.apache.hadoop.yarn.server.federation.store.impl;
import java.io.IOException;
-import org.junit.Before;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
/**
@@ -45,17 +41,22 @@ import org.junit.Test;
*/
public abstract class FederationStateStoreBaseTest {
- static final Logger LOG =
- LoggerFactory.getLogger(FederationStateStoreBaseTest.class);
private static final MonotonicClock CLOCK = new MonotonicClock();
- private FederationMembershipStateStore stateStore = getStateStore();
+ private FederationMembershipStateStore stateStore;
@Before
public void before() throws IOException {
- clearMembership();
+ stateStore = getCleanStateStore();
}
+ @After
+ public void after() {
+ stateStore = null;
+ }
+
+ protected abstract FederationMembershipStateStore getCleanStateStore();
+
@Test
public void testRegisterSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
@@ -63,11 +64,9 @@ public abstract class FederationStateStoreBaseTest {
SubClusterRegisterResponse result = stateStore.registerSubCluster(
SubClusterRegisterRequest.newInstance(subClusterInfo));
- Map<SubClusterId, SubClusterInfo> membership = getMembership();
- Assert.assertNotNull(membership.get(subClusterId));
Assert.assertNotNull(result);
- Assert.assertEquals(subClusterInfo, membership.get(subClusterId));
+ Assert.assertEquals(subClusterInfo, querySubClusterInfo(subClusterId));
}
@Test
@@ -83,10 +82,8 @@ public abstract class FederationStateStoreBaseTest {
stateStore.deregisterSubCluster(deregisterRequest);
- Map<SubClusterId, SubClusterInfo> membership = getMembership();
- Assert.assertNotNull(membership.get(subClusterId));
- Assert.assertEquals(membership.get(subClusterId).getState(),
- SubClusterState.SC_UNREGISTERED);
+ Assert.assertEquals(SubClusterState.SC_UNREGISTERED,
+ querySubClusterInfo(subClusterId).getState());
}
@Test
@@ -179,10 +176,9 @@ public abstract class FederationStateStoreBaseTest {
.newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
stateStore.subClusterHeartbeat(heartbeatRequest);
- Map<SubClusterId, SubClusterInfo> membership = getMembership();
- Assert.assertEquals(membership.get(subClusterId).getState(),
- SubClusterState.SC_RUNNING);
- Assert.assertNotNull(membership.get(subClusterId).getLastHeartBeat());
+ Assert.assertEquals(SubClusterState.SC_RUNNING,
+ querySubClusterInfo(subClusterId).getState());
+ Assert.assertNotNull(querySubClusterInfo(subClusterId).getLastHeartBeat());
}
@Test
@@ -212,10 +208,11 @@ public abstract class FederationStateStoreBaseTest {
CLOCK.getTime(), "cabability");
}
- protected abstract Map<SubClusterId, SubClusterInfo> getMembership();
-
- protected abstract void clearMembership();
-
- protected abstract FederationMembershipStateStore getStateStore();
+ private SubClusterInfo querySubClusterInfo(SubClusterId subClusterId)
+ throws YarnException {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ return stateStore.getSubCluster(request).getSubClusterInfo();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca9c2236/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
index b74ffbd..9396eda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -17,11 +17,7 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
-import java.util.Map;
-
import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
/**
* Unit tests for MemoryFederationStateStore.
@@ -29,21 +25,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
public class TestMemoryFederationStateStore
extends FederationStateStoreBaseTest {
- private static final MemoryFederationStateStore STATESTORE =
- new MemoryFederationStateStore();
-
- @Override
- protected Map<SubClusterId, SubClusterInfo> getMembership() {
- return STATESTORE.getMembershipTable();
- }
-
- @Override
- protected void clearMembership() {
- STATESTORE.clearMembershipTable();
- }
-
@Override
- protected FederationMembershipStateStore getStateStore() {
- return STATESTORE;
+ protected FederationMembershipStateStore getCleanStateStore() {
+ return new MemoryFederationStateStore();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-14113. Review ADL Docs.
Contributed by Steve Loughran
Posted by su...@apache.org.
HADOOP-14113. Review ADL Docs. Contributed by Steve Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e60c6543
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e60c6543
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e60c6543
Branch: refs/heads/YARN-2915
Commit: e60c6543d57611039b0438d5dcb4cb19ee239bb6
Parents: 9c22a91
Author: Steve Loughran <st...@apache.org>
Authored: Fri Feb 24 13:24:59 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Feb 24 13:24:59 2017 +0000
----------------------------------------------------------------------
.../src/site/markdown/index.md | 237 ++++++++++---------
1 file changed, 124 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e60c6543/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index 6d9e173..9355241 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -20,20 +20,20 @@
* [Usage](#Usage)
* [Concepts](#Concepts)
* [OAuth2 Support](#OAuth2_Support)
- * [Configuring Credentials & FileSystem](#Configuring_Credentials)
+ * [Configuring Credentials and FileSystem](#Configuring_Credentials)
* [Using Refresh Token](#Refresh_Token)
* [Using Client Keys](#Client_Credential_Token)
* [Protecting the Credentials with Credential Providers](#Credential_Provider)
* [Enabling ADL Filesystem](#Enabling_ADL)
- * [Accessing adl URLs](#Accessing_adl_URLs)
+ * [Accessing `adl` URLs](#Accessing_adl_URLs)
* [User/Group Representation](#OIDtoUPNConfiguration)
-* [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
+* [Testing the `hadoop-azure` Module](#Testing_the_hadoop-azure_Module)
## <a name="Introduction" />Introduction
-The hadoop-azure-datalake module provides support for integration with
-[Azure Data Lake Store]( https://azure.microsoft.com/en-in/documentation/services/data-lake-store/).
-The jar file is named azure-datalake-store.jar.
+The `hadoop-azure-datalake` module provides support for integration with the
+[Azure Data Lake Store](https://azure.microsoft.com/en-in/documentation/services/data-lake-store/).
+This support comes via the JAR file `azure-datalake-store.jar`.
## <a name="Features" />Features
@@ -43,13 +43,14 @@ The jar file is named azure-datalake-store.jar.
* Can act as a source of data in a MapReduce job, or a sink.
* Tested on both Linux and Windows.
* Tested for scale.
-* API setOwner/setAcl/removeAclEntries/modifyAclEntries accepts UPN or OID
- (Object ID) as user and group name.
+* API `setOwner()`, `setAcl`, `removeAclEntries()`, `modifyAclEntries()` accepts UPN or OID
+ (Object ID) as user and group names.
## <a name="Limitations" />Limitations
+
Partial or no support for the following operations :
-* Operation on Symbolic Link
+* Operation on Symbolic Links
* Proxy Users
* File Truncate
* File Checksum
@@ -58,55 +59,71 @@ Partial or no support for the following operations :
* Extended Attributes(XAttrs) Operations
* Snapshot Operations
* Delegation Token Operations
-* User and group information returned as ListStatus and GetFileStatus is in form of GUID associated in Azure Active Directory.
+* User and group information returned as `listStatus()` and `getFileStatus()` is
+in the form of the GUID associated in Azure Active Directory.
## <a name="Usage" />Usage
### <a name="Concepts" />Concepts
-Azure Data Lake Storage access path syntax is
+Azure Data Lake Storage access path syntax is:
+
+```
+adl://<Account Name>.azuredatalakestore.net/
+```
- adl://<Account Name>.azuredatalakestore.net/
+For details on using the store, see
+[**Get started with Azure Data Lake Store using the Azure Portal**](https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/)
-Get started with azure data lake account with [https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/](https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/)
+### <a name="#OAuth2_Support" />OAuth2 Support
-#### <a name="#OAuth2_Support" />OAuth2 Support
-Usage of Azure Data Lake Storage requires OAuth2 bearer token to be present as part of the HTTPS header as per OAuth2 specification. Valid OAuth2 bearer token should be obtained from Azure Active Directory for valid users who have access to Azure Data Lake Storage Account.
+Usage of Azure Data Lake Storage requires an OAuth2 bearer token to be present as
+part of the HTTPS header as per the OAuth2 specification.
+A valid OAuth2 bearer token must be obtained from the Azure Active Directory service
+for those valid users who have access to Azure Data Lake Storage Account.
-Azure Active Directory (Azure AD) is Microsoft's multi-tenant cloud based directory and identity management service. See [https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/](https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/)
+Azure Active Directory (Azure AD) is Microsoft's multi-tenant cloud based directory
+and identity management service. See [*What is ActiveDirectory*](https://azure.microsoft.com/en-in/documentation/articles/active-directory-whatis/).
-Following sections describes on OAuth2 configuration in core-site.xml.
+Following sections describes theOAuth2 configuration in `core-site.xml`.
-## <a name="Configuring_Credentials" />Configuring Credentials & FileSystem
-Credentials can be configured using either a refresh token (associated with a user) or a client credential (analogous to a service principal).
+#### <a name="Configuring_Credentials" />Configuring Credentials & FileSystem
+Credentials can be configured using either a refresh token (associated with a user),
+or a client credential (analogous to a service principal).
-### <a name="Refresh_Token" />Using Refresh Token
+#### <a name="Refresh_Token" />Using Refresh Tokens
-Add the following properties to your core-site.xml
+Add the following properties to the cluster's `core-site.xml`
- <property>
- <name>dfs.adls.oauth2.access.token.provider.type</name>
- <value>RefreshToken</value>
- </property>
+```xml
+<property>
+ <name>dfs.adls.oauth2.access.token.provider.type</name>
+ <value>RefreshToken</value>
+</property>
+```
-Application require to set Client id and OAuth2 refresh token from Azure Active Directory associated with client id. See [https://github.com/AzureAD/azure-activedirectory-library-for-java](https://github.com/AzureAD/azure-activedirectory-library-for-java).
+Applications must set the Client id and OAuth2 refresh token from the Azure Active Directory
+service associated with the client id. See [*Active Directory Library For Java*](https://github.com/AzureAD/azure-activedirectory-library-for-java).
**Do not share client id and refresh token, it must be kept secret.**
- <property>
- <name>dfs.adls.oauth2.client.id</name>
- <value></value>
- </property>
+```xml
+<property>
+ <name>dfs.adls.oauth2.client.id</name>
+ <value></value>
+</property>
- <property>
- <name>dfs.adls.oauth2.refresh.token</name>
- <value></value>
- </property>
+<property>
+ <name>dfs.adls.oauth2.refresh.token</name>
+ <value></value>
+</property>
+```
### <a name="Client_Credential_Token" />Using Client Keys
#### Generating the Service Principal
-1. Go to the portal (https://portal.azure.com)
+
+1. Go to [the portal](https://portal.azure.com)
2. Under "Browse", look for Active Directory and click on it.
3. Create "Web Application". Remember the name you create here - that is what you will add to your ADL account as authorized user.
4. Go through the wizard
@@ -124,31 +141,31 @@ Application require to set Client id and OAuth2 refresh token from Azure Active
3. Add your user name you created in Step 6 above (note that it does not show up in the list, but will be found if you searched for the name)
4. Add "Owner" role
-#### Configure core-site.xml
-Add the following properties to your core-site.xml
-
- <property>
- <name>dfs.adls.oauth2.refresh.url</name>
- <value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
- </property>
+### Configure core-site.xml
+Add the following properties to your `core-site.xml`
- <property>
- <name>dfs.adls.oauth2.client.id</name>
- <value>CLIENT ID FROM STEP 7 ABOVE</value>
- </property>
+```xml
+<property>
+ <name>dfs.adls.oauth2.refresh.url</name>
+ <value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
+</property>
- <property>
- <name>dfs.adls.oauth2.credential</name>
- <value>PASSWORD FROM STEP 7 ABOVE</value>
- </property>
+<property>
+ <name>dfs.adls.oauth2.client.id</name>
+ <value>CLIENT ID FROM STEP 7 ABOVE</value>
+</property>
+<property>
+ <name>dfs.adls.oauth2.credential</name>
+ <value>PASSWORD FROM STEP 7 ABOVE</value>
+</property>
+```
### <a name="Credential_Provider" />Protecting the Credentials with Credential Providers
-In many Hadoop clusters, the core-site.xml file is world-readable. To protect
-these credentials from prying eyes, it is recommended that you use the
-credential provider framework to securely store them and access them through
-configuration.
+In many Hadoop clusters, the `core-site.xml` file is world-readable. To protect
+these credentials, it is recommended that you use the
+credential provider framework to securely store them and access them.
All ADLS credential properties can be protected by credential providers.
For additional reading on the credential provider API, see
@@ -156,16 +173,16 @@ For additional reading on the credential provider API, see
#### Provisioning
-```
-% hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+```bash
+hadoop credential create dfs.adls.oauth2.refresh.token -value 123
-provider localjceks://file/home/foo/adls.jceks
-% hadoop credential create dfs.adls.oauth2.credential -value 123
+hadoop credential create dfs.adls.oauth2.credential -value 123
-provider localjceks://file/home/foo/adls.jceks
```
#### Configuring core-site.xml or command line property
-```
+```xml
<property>
<name>hadoop.security.credential.provider.path</name>
<value>localjceks://file/home/foo/adls.jceks</value>
@@ -175,42 +192,28 @@ For additional reading on the credential provider API, see
#### Running DistCp
-```
-% hadoop distcp
+```bash
+hadoop distcp
[-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
hdfs://<NameNode Hostname>:9001/user/foo/007020615
adl://<Account Name>.azuredatalakestore.net/testDir/
```
-NOTE: You may optionally add the provider path property to the distcp command
-line instead of added job specific configuration to a generic core-site.xml.
-The square brackets above illustrate this capability.
-
-
-## <a name="Enabling_ADL" />Enabling ADL Filesystem
-
-For ADL FileSystem to take effect. Update core-site.xml with
-
- <property>
- <name>fs.adl.impl</name>
- <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
- </property>
-
- <property>
- <name>fs.AbstractFileSystem.adl.impl</name>
- <value>org.apache.hadoop.fs.adl.Adl</value>
- </property>
-
+NOTE: You may optionally add the provider path property to the `distcp` command
+line instead of added job specific configuration to a generic `core-site.xml`.
+The square brackets above illustrate this capability.`
### <a name="Accessing_adl_URLs" />Accessing adl URLs
-After credentials are configured in core-site.xml, any Hadoop component may
+After credentials are configured in `core-site.xml`, any Hadoop component may
reference files in that Azure Data Lake Storage account by using URLs of the following
format:
- adl://<Account Name>.azuredatalakestore.net/<path>
+```
+adl://<Account Name>.azuredatalakestore.net/<path>
+```
-The schemes `adl` identify a URL on a file system backed by Azure
+The schemes `adl` identifies a URL on a Hadoop-compatible file system backed by Azure
Data Lake Storage. `adl` utilizes encrypted HTTPS access for all interaction with
the Azure Data Lake Storage API.
@@ -218,48 +221,56 @@ For example, the following
[FileSystem Shell](../hadoop-project-dist/hadoop-common/FileSystemShell.html)
commands demonstrate access to a storage account named `youraccount`.
- > hadoop fs -mkdir adl://yourcontainer.azuredatalakestore.net/testDir
- > hadoop fs -put testFile adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+```bash
+hadoop fs -mkdir adl://yourcontainer.azuredatalakestore.net/testDir
- > hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
- test file content
+hadoop fs -put testFile adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
+test file content
+```
### <a name="OIDtoUPNConfiguration" />User/Group Representation
-The hadoop-azure-datalake module provides support for configuring how
-User/Group information is represented during
-getFileStatus/listStatus/getAclStatus.
-Add the following properties to your core-site.xml
-
- <property>
- <name>adl.feature.ownerandgroup.enableupn</name>
- <value>true</value>
- <description>
- When true : User and Group in FileStatus/AclStatus response is
- represented as user friendly name as per Azure AD profile.
-
- When false (default) : User and Group in FileStatus/AclStatus
- response is represented by the unique identifier from Azure AD
- profile (Object ID as GUID).
+The `hadoop-azure-datalake` module provides support for configuring how
+User/Group information is represented during
+`getFileStatus()`, `listStatus()`, and `getAclStatus()` calls..
- For performance optimization, Recommended default value.
- </description>
- </property>
+Add the following properties to `core-site.xml`
+```xml
+<property>
+ <name>adl.feature.ownerandgroup.enableupn</name>
+ <value>true</value>
+ <description>
+ When true : User and Group in FileStatus/AclStatus response is
+ represented as user friendly name as per Azure AD profile.
+
+ When false (default) : User and Group in FileStatus/AclStatus
+ response is represented by the unique identifier from Azure AD
+ profile (Object ID as GUID).
+
+ For performance optimization, Recommended default value.
+ </description>
+</property>
+```
## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
-The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
+The `hadoop-azure` module includes a full suite of unit tests.
+Most of the tests will run without additional configuration by running mvn test.
+This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
A selection of tests can run against the Azure Data Lake Storage. To run these
tests, please create `src/test/resources/auth-keys.xml` with Adl account
information mentioned in the above sections and the following properties.
- <property>
- <name>dfs.adl.test.contract.enable</name>
- <value>true</value>
- </property>
+```xml
+<property>
+ <name>dfs.adl.test.contract.enable</name>
+ <value>true</value>
+</property>
- <property>
- <name>test.fs.adl.name</name>
- <value>adl://yourcontainer.azuredatalakestore.net</value>
- </property>
+<property>
+ <name>test.fs.adl.name</name>
+ <value>adl://yourcontainer.azuredatalakestore.net</value>
+</property>
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-5323. Policies APIs for
Federation Router and AMRMProxy policies. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5323. Policies APIs for Federation Router and AMRMProxy policies. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/088ed959
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/088ed959
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/088ed959
Branch: refs/heads/YARN-2915
Commit: 088ed959c785a1535572a128596345f780412878
Parents: d4dd6de
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Sep 7 17:33:34 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../policies/ConfigurableFederationPolicy.java | 44 +++++++
.../policies/FederationPolicyConfigurator.java | 91 +++++++++++++
.../FederationPolicyInitializationContext.java | 109 ++++++++++++++++
...ionPolicyInitializationContextValidator.java | 82 ++++++++++++
.../policies/FederationPolicyWriter.java | 45 +++++++
.../amrmproxy/FederationAMRMProxyPolicy.java | 66 ++++++++++
.../policies/amrmproxy/package-info.java | 20 +++
.../exceptions/FederationPolicyException.java | 33 +++++
...FederationPolicyInitializationException.java | 33 +++++
.../NoActiveSubclustersException.java | 27 ++++
.../exceptions/UnknownSubclusterException.java | 28 ++++
.../policies/exceptions/package-info.java | 20 +++
.../federation/policies/package-info.java | 20 +++
.../policies/router/FederationRouterPolicy.java | 45 +++++++
.../policies/router/package-info.java | 20 +++
...ionPolicyInitializationContextValidator.java | 128 +++++++++++++++++++
.../utils/FederationPoliciesTestUtil.java | 83 ++++++++++++
17 files changed, 894 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
new file mode 100644
index 0000000..fd6ceea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+/**
+ * This interface provides a general method to reinitialize a policy. The
+ * semantics are try-n-swap, so in case of an exception is thrown the
+ * implmentation must ensure the previous state and configuration is preserved.
+ */
+public interface ConfigurableFederationPolicy {
+
+ /**
+ * This method is invoked to initialize of update the configuration of
+ * policies. The implementor should provide try-n-swap semantics, and retain
+ * state if possible.
+ *
+ * @param federationPolicyInitializationContext the new context to provide to
+ * implementor.
+ *
+ * @throws FederationPolicyInitializationException in case the initialization
+ * fails.
+ */
+ void reinitialize(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext)
+ throws FederationPolicyInitializationException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
new file mode 100644
index 0000000..fdc3857
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+import org.apache.hadoop.yarn.server.federation.policies.router
+ .FederationRouterPolicy;
+
+/**
+ * Implementors of this interface are capable to instantiate and (re)initalize
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} based on
+ * a {@link FederationPolicyInitializationContext}. The reason to bind these two
+ * policies together is to make sure we remain consistent across the router and
+ * amrmproxy policy decisions.
+ */
+public interface FederationPolicyConfigurator {
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
+ * current context, otherwise a new instance initialized with the current
+ * context is provided. If the instance is compatible with the current class
+ * the implementors should attempt to reinitalize (retaining state). To affect
+ * a complete policy reset oldInstance should be null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationAMRMProxyPolicy
+ }.
+ *
+ * @throws FederationPolicyInitializationException if the initialization
+ * cannot be completed
+ * properly. The oldInstance
+ * should be still valid in
+ * case of failed
+ * initialization.
+ */
+ FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationRouterPolicy} reinitialized with the current
+ * context, otherwise a new instance initialized with the current context is
+ * provided. If the instance is compatible with the current class the
+ * implementors should attempt to reinitalize (retaining state). To affect a
+ * complete policy reset oldInstance shoulb be set to null.
+ *
+ * @param federationPolicyInitializationContext the current context
+ * @param oldInstance the existing (possibly null)
+ * instance.
+ *
+ * @return an updated {@link FederationRouterPolicy}.
+ *
+ * @throws FederationPolicyInitializationException if the initalization cannot
+ * be completed properly. The
+ * oldInstance should be still
+ * valid in case of failed
+ * initialization.
+ */
+ FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
new file mode 100644
index 0000000..879ccee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContext.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+
+/**
+ * Context to (re)initialize a {@code FederationAMRMProxyPolicy} and {@code
+ * FederationRouterPolicy}.
+ */
+public class FederationPolicyInitializationContext {
+
+ private SubClusterPolicyConfiguration federationPolicyConfiguration;
+ private SubClusterResolver federationSubclusterResolver;
+ private FederationStateStoreFacade federationStateStoreFacade;
+
+ public FederationPolicyInitializationContext() {
+ federationPolicyConfiguration = null;
+ federationSubclusterResolver = null;
+ federationStateStoreFacade = null;
+ }
+
+ public FederationPolicyInitializationContext(SubClusterPolicyConfiguration
+ policy, SubClusterResolver resolver, FederationStateStoreFacade
+ storeFacade) {
+ this.federationPolicyConfiguration = policy;
+ this.federationSubclusterResolver = resolver;
+ this.federationStateStoreFacade = storeFacade;
+ }
+
+
+ /**
+ * Getter for the {@link SubClusterPolicyConfiguration}.
+ *
+ * @return the {@link SubClusterPolicyConfiguration} to be used for
+ * initialization.
+ */
+ public SubClusterPolicyConfiguration getSubClusterPolicyConfiguration() {
+ return federationPolicyConfiguration;
+ }
+
+ /**
+ * Setter for the {@link SubClusterPolicyConfiguration}.
+ *
+ * @param federationPolicyConfiguration the
+ * {@link SubClusterPolicyConfiguration}
+ * to be used for initialization.
+ */
+ public void setFederationPolicyConfiguration(
+ SubClusterPolicyConfiguration federationPolicyConfiguration) {
+ this.federationPolicyConfiguration = federationPolicyConfiguration;
+ }
+
+ /**
+ * Getter for the {@link SubClusterResolver}.
+ *
+ * @return the {@link SubClusterResolver} to be used for initialization.
+ */
+ public SubClusterResolver getFederationSubclusterResolver() {
+ return federationSubclusterResolver;
+ }
+
+ /**
+ * Setter for the {@link SubClusterResolver}.
+ *
+ * @param federationSubclusterResolver the {@link SubClusterResolver} to be
+ * used for initialization.
+ */
+ public void setFederationSubclusterResolver(
+ SubClusterResolver federationSubclusterResolver) {
+ this.federationSubclusterResolver = federationSubclusterResolver;
+ }
+
+ /**
+ * Getter for the {@link FederationStateStoreFacade}.
+ *
+ * @return the facade.
+ */
+ public FederationStateStoreFacade getFederationStateStoreFacade() {
+ return federationStateStoreFacade;
+ }
+
+ /**
+ * Setter for the {@link FederationStateStoreFacade}.
+ *
+ * @param federationStateStoreFacade the facade.
+ */
+ public void setFederationStateStoreFacade(
+ FederationStateStoreFacade federationStateStoreFacade) {
+ this.federationStateStoreFacade = federationStateStoreFacade;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
new file mode 100644
index 0000000..31f83d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyInitializationContextValidator.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+/**
+ * Helper class used to factor out common validation steps for policies.
+ */
+public final class FederationPolicyInitializationContextValidator {
+
+ private FederationPolicyInitializationContextValidator() {
+ //disable constructor per checkstyle
+ }
+
+ public static void validate(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ String myType) throws FederationPolicyInitializationException {
+
+ if (myType == null) {
+ throw new FederationPolicyInitializationException("The myType parameter"
+ + " should not be null.");
+ }
+
+ if (federationPolicyInitializationContext == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationPolicyInitializationContext provided is null. Cannot"
+ + " reinitalize "
+ + "successfully.");
+ }
+
+ if (federationPolicyInitializationContext.getFederationStateStoreFacade()
+ == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationStateStoreFacade provided is null. Cannot"
+ + " reinitalize successfully.");
+ }
+
+ if (federationPolicyInitializationContext.getFederationSubclusterResolver()
+ == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationStateStoreFacase provided is null. Cannot"
+ + " reinitalize successfully.");
+ }
+
+ if (federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
+ == null) {
+ throw new FederationPolicyInitializationException(
+ "The FederationSubclusterResolver provided is null. Cannot "
+ + "reinitalize successfully.");
+ }
+
+ String intendedType =
+ federationPolicyInitializationContext.getSubClusterPolicyConfiguration()
+ .getType();
+
+ if (!myType.equals(intendedType)) {
+ throw new FederationPolicyInitializationException(
+ "The FederationPolicyConfiguration carries a type (" + intendedType
+ + ") different then mine (" + myType
+ + "). Cannot reinitialize successfully.");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
new file mode 100644
index 0000000..5034b7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyWriter.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+/**
+ * Implementors of this class are able to serializeConf the configuraiton of a
+ * policy as a {@link SubClusterPolicyConfiguration}. This is used during the
+ * lifetime of a policy from the admin APIs or policy engine to serializeConf
+ * the policy into the policy store.
+ */
+public interface FederationPolicyWriter {
+
+ /**
+ /**
+ * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
+ * This is to be used when writing a policy object in the federation policy
+ * store.
+ *
+ * @return a valid policy configuration representing this object
+ * parametrization.
+ *
+ * @throws FederationPolicyInitializationException if the current state cannot
+ * be serialized properly
+ */
+ SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
new file mode 100644
index 0000000..4a3305c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Implementors of this interface provide logic to split the list of {@link
+ * ResourceRequest}s received by the AM among various RMs.
+ */
+public interface FederationAMRMProxyPolicy
+ extends ConfigurableFederationPolicy {
+
+ /**
+ * Splits the {@link ResourceRequest}s from the client across one or more
+ * sub-clusters based on the policy semantics (e.g., broadcast, load-based).
+ *
+ * @param resourceRequests the list of {@link ResourceRequest}s from the
+ * AM to be split
+ *
+ * @return map of sub-cluster as identified by {@link SubClusterId} to the
+ * list of {@link ResourceRequest}s that should be forwarded to it
+ *
+ * @throws YarnException in case the request is malformed or no viable
+ * sub-clusters can be found.
+ */
+ Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests)
+ throws YarnException;
+
+ /**
+ * This method should be invoked to notify the policy about responses being
+ * received. This is useful for stateful policies that make decisions based on
+ * previous responses being received.
+ *
+ * @param subClusterId the id of the subcluster sending the notification
+ * @param response the response received from one of the RMs
+ *
+ * @throws YarnException in case the response is not valid
+ */
+ void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
new file mode 100644
index 0000000..99da20b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** AMRMPRoxy policies. **/
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java
new file mode 100644
index 0000000..24fe421
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Generic policy exception.
+ */
+public class FederationPolicyException extends YarnException {
+ public FederationPolicyException(String s) {
+ super(s);
+ }
+
+ public FederationPolicyException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java
new file mode 100644
index 0000000..fcc09c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/FederationPolicyInitializationException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+/**
+ * This exception is thrown when the initialization of a federation policy is
+ * not successful.
+ */
+public class FederationPolicyInitializationException
+ extends FederationPolicyException {
+ public FederationPolicyInitializationException(String message) {
+ super(message);
+ }
+
+ public FederationPolicyInitializationException(Throwable j) {
+ super(j);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java
new file mode 100644
index 0000000..a427944
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/NoActiveSubclustersException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+/**
+ * This exception is thrown when policies cannot locate any active cluster.
+ */
+public class NoActiveSubclustersException extends FederationPolicyException {
+ public NoActiveSubclustersException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java
new file mode 100644
index 0000000..8a0fb4f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/UnknownSubclusterException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
+/**
+ * This exception is thrown whenever a policy is given a {@code SubClusterId}
+ * that is unknown.
+ */
+public class UnknownSubclusterException extends FederationPolicyException {
+ public UnknownSubclusterException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
new file mode 100644
index 0000000..3318da9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/exceptions/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Exceptions for policies. **/
+package org.apache.hadoop.yarn.server.federation.policies.exceptions;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
new file mode 100644
index 0000000..7d9a121
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Federation Policies. **/
+package org.apache.hadoop.yarn.server.federation.policies;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
new file mode 100644
index 0000000..42c86cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * Implements the logic for determining the routing of an application submission
+ * based on a policy.
+ */
+public interface FederationRouterPolicy extends ConfigurableFederationPolicy {
+
+ /**
+ * Determines the sub-cluster that the user application submision should be
+ * routed to.
+ *
+ * @param appSubmissionContext the context for the app being submitted.
+ *
+ * @return the sub-cluster as identified by {@link SubClusterId} to route the
+ * request to.
+ *
+ * @throws YarnException if the policy cannot determine a viable subcluster.
+ */
+ SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext)
+ throws YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
new file mode 100644
index 0000000..5d0fcb6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Router policies. **/
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
new file mode 100644
index 0000000..4ec04d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Test class for {@link FederationPolicyInitializationContextValidator}.
+ */
+public class TestFederationPolicyInitializationContextValidator {
+
+ private SubClusterPolicyConfiguration goodConfig;
+ private SubClusterResolver goodSR;
+ private FederationStateStoreFacade goodFacade;
+ private FederationPolicyInitializationContext context;
+
+ @Before
+ public void setUp() throws Exception {
+ goodFacade = FederationPoliciesTestUtil.initFacade();
+ goodConfig =
+ new MockPolicyManager().serializeConf();
+ goodSR =FederationPoliciesTestUtil.initResolver();
+ context = new
+ FederationPolicyInitializationContext(goodConfig, goodSR, goodFacade);
+ }
+
+ @Test
+ public void correcInit() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullContext() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(null,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullType() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(context, null);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void wrongType() throws Exception {
+ FederationPolicyInitializationContextValidator.validate(context,
+ "WrongType");
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullConf() throws Exception {
+ context.setFederationPolicyConfiguration(null);
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullResolver() throws Exception {
+ context.setFederationSubclusterResolver(null);
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void nullFacade() throws Exception {
+ context.setFederationStateStoreFacade(null);
+ FederationPolicyInitializationContextValidator.validate(context,
+ MockPolicyManager.class.getCanonicalName());
+ }
+
+ private class MockPolicyManager
+ implements FederationPolicyWriter, FederationPolicyConfigurator {
+ @Override
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+ return null;
+ }
+
+ @Override
+ public FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+ return null;
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = ByteBuffer.allocate(0);
+ return SubClusterPolicyConfiguration
+ .newInstance("queue1", this.getClass().getCanonicalName(), buf);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ed959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
new file mode 100644
index 0000000..8c2115b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
+import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+
+import java.net.URL;
+import java.util.ArrayList;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Support class providing common initialization methods to test federation
+ * policies.
+ */
+public final class FederationPoliciesTestUtil {
+
+ private FederationPoliciesTestUtil() {
+ // disabled.
+ }
+
+ /**
+ * Initialize a {@link SubClusterResolver}.
+ *
+ * @return a subcluster resolver for tests.
+ */
+ public static SubClusterResolver initResolver() {
+ YarnConfiguration conf = new YarnConfiguration();
+ SubClusterResolver resolver =
+ new DefaultSubClusterResolverImpl();
+ URL url =
+ Thread.currentThread().getContextClassLoader().getResource("nodes");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'nodes' dummy file in classpath");
+ }
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, url.getPath());
+ resolver.setConf(conf);
+ resolver.load();
+ return resolver;
+ }
+
+ /**
+ * Initialiaze a main-memory {@link FederationStateStoreFacade} used for
+ * testing, wiht a mock resolver.
+ *
+ * @return the facade.
+ *
+ * @throws YarnException in case the initialization is not successful.
+ */
+ public static FederationStateStoreFacade initFacade() throws YarnException {
+ FederationStateStoreFacade goodFacade = FederationStateStoreFacade
+ .getInstance();
+ FederationStateStore fss = mock(FederationStateStore.class);
+ GetSubClustersInfoResponse response = GetSubClustersInfoResponse
+ .newInstance(new ArrayList<>());
+ when(fss.getSubClusters(any())).thenReturn(response);
+ goodFacade.reinitialize(fss, new Configuration());
+ return goodFacade;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-5467. InputValidator for the
FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
Posted by su...@apache.org.
YARN-5467. InputValidator for the FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23d57781
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23d57781
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23d57781
Branch: refs/heads/YARN-2915
Commit: 23d5778104bb042aa614c1f5751cd5c731ba724f
Parents: c73ed1f
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Aug 17 12:07:06 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../store/impl/MemoryFederationStateStore.java | 30 +
...cationHomeSubClusterStoreInputValidator.java | 183 +++
...ationMembershipStateStoreInputValidator.java | 317 +++++
.../FederationPolicyStoreInputValidator.java | 144 ++
...derationStateStoreInvalidInputException.java | 48 +
.../federation/store/utils/package-info.java | 17 +
.../impl/FederationStateStoreBaseTest.java | 6 +-
.../TestFederationStateStoreInputValidator.java | 1265 ++++++++++++++++++
8 files changed, 2007 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 8144435..6e564dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -57,6 +57,9 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegister
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
@@ -88,6 +91,8 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterRegisterRequest(request);
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
return SubClusterRegisterResponse.newInstance();
@@ -96,6 +101,8 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public SubClusterDeregisterResponse deregisterSubCluster(
SubClusterDeregisterRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterDeregisterRequest(request);
SubClusterInfo subClusterInfo = membership.get(request.getSubClusterId());
if (subClusterInfo == null) {
throw new YarnException(
@@ -111,6 +118,8 @@ public class MemoryFederationStateStore implements FederationStateStore {
public SubClusterHeartbeatResponse subClusterHeartbeat(
SubClusterHeartbeatRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator
+ .validateSubClusterHeartbeatRequest(request);
SubClusterId subClusterId = request.getSubClusterId();
SubClusterInfo subClusterInfo = membership.get(subClusterId);
@@ -129,6 +138,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public GetSubClusterInfoResponse getSubCluster(
GetSubClusterInfoRequest request) throws YarnException {
+
+ FederationMembershipStateStoreInputValidator
+ .validateGetSubClusterInfoRequest(request);
SubClusterId subClusterId = request.getSubClusterId();
if (!membership.containsKey(subClusterId)) {
throw new YarnException(
@@ -157,6 +169,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateAddApplicationHomeSubClusterRequest(request);
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
@@ -172,6 +187,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateUpdateApplicationHomeSubClusterRequest(request);
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
if (!applications.containsKey(appId)) {
@@ -186,6 +204,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateGetApplicationHomeSubClusterRequest(request);
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
throw new YarnException("Application " + appId + " does not exist");
@@ -212,6 +233,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
DeleteApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator
+ .validateDeleteApplicationHomeSubClusterRequest(request);
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
throw new YarnException("Application " + appId + " does not exist");
@@ -224,6 +248,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+
+ FederationPolicyStoreInputValidator
+ .validateGetSubClusterPolicyConfigurationRequest(request);
String queue = request.getQueue();
if (!policies.containsKey(queue)) {
throw new YarnException("Policy for queue " + queue + " does not exist");
@@ -236,6 +263,9 @@ public class MemoryFederationStateStore implements FederationStateStore {
@Override
public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+
+ FederationPolicyStoreInputValidator
+ .validateSetSubClusterPolicyConfigurationRequest(request);
policies.put(request.getPolicyConfiguration().getQueue(),
request.getPolicyConfiguration());
return SetSubClusterPolicyConfigurationResponse.newInstance();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java
new file mode 100644
index 0000000..c14a452
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationApplicationHomeSubClusterStoreInputValidator.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to
+ * {@code FederationApplicationHomeSubClusterStore}, allows a fail fast
+ * mechanism for invalid user inputs.
+ *
+ */
+public final class FederationApplicationHomeSubClusterStoreInputValidator {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(FederationApplicationHomeSubClusterStoreInputValidator.class);
+
+ private FederationApplicationHomeSubClusterStoreInputValidator() {
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link AddApplicationHomeSubClusterRequest}
+ * for adding a new application is valid or not.
+ *
+ * @param request the {@link AddApplicationHomeSubClusterRequest} to validate
+ * against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateAddApplicationHomeSubClusterRequest(
+ AddApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing AddApplicationHomeSubCluster Request."
+ + " Please try again by specifying"
+ + " an AddApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate ApplicationHomeSubCluster info
+ checkApplicationHomeSubCluster(request.getApplicationHomeSubCluster());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link UpdateApplicationHomeSubClusterRequest}
+ * for updating an application is valid or not.
+ *
+ * @param request the {@link UpdateApplicationHomeSubClusterRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateUpdateApplicationHomeSubClusterRequest(
+ UpdateApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing UpdateApplicationHomeSubCluster Request."
+ + " Please try again by specifying"
+ + " an ApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate ApplicationHomeSubCluster info
+ checkApplicationHomeSubCluster(request.getApplicationHomeSubCluster());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link GetApplicationHomeSubClusterRequest}
+ * for querying application's information is valid or not.
+ *
+ * @param request the {@link GetApplicationHomeSubClusterRequest} to validate
+ * against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateGetApplicationHomeSubClusterRequest(
+ GetApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing GetApplicationHomeSubCluster Request."
+ + " Please try again by specifying an Application Id information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate application Id
+ checkApplicationId(request.getApplicationId());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link DeleteApplicationHomeSubClusterRequest}
+ * for deleting an application is valid or not.
+ *
+ * @param request the {@link DeleteApplicationHomeSubClusterRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateDeleteApplicationHomeSubClusterRequest(
+ DeleteApplicationHomeSubClusterRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing DeleteApplicationHomeSubCluster Request."
+ + " Please try again by specifying"
+ + " an ApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate application Id
+ checkApplicationId(request.getApplicationId());
+ }
+
+ /**
+ * Validate if the ApplicationHomeSubCluster info are present or not.
+ *
+ * @param applicationHomeSubCluster the information of the application to be
+ * verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster Info
+ * are invalid
+ */
+ private static void checkApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster)
+
+ throws FederationStateStoreInvalidInputException {
+ if (applicationHomeSubCluster == null) {
+ String message = "Missing ApplicationHomeSubCluster Info."
+ + " Please try again by specifying"
+ + " an ApplicationHomeSubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ // validate application Id
+ checkApplicationId(applicationHomeSubCluster.getApplicationId());
+
+ // validate subcluster Id
+ FederationMembershipStateStoreInputValidator
+ .checkSubClusterId(applicationHomeSubCluster.getHomeSubCluster());
+
+ }
+
+ /**
+ * Validate if the application id is present or not.
+ *
+ * @param appId the id of the application to be verified
+ * @throws FederationStateStoreInvalidInputException if the application Id is
+ * invalid
+ */
+ private static void checkApplicationId(ApplicationId appId)
+ throws FederationStateStoreInvalidInputException {
+ if (appId == null) {
+ String message = "Missing Application Id."
+ + " Please try again by specifying an Application Id.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
new file mode 100644
index 0000000..b587ee5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import java.net.URI;
+
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to
+ * {@code FederationMembershipStateStore}, allows a fail fast mechanism for
+ * invalid user inputs.
+ *
+ */
+public final class FederationMembershipStateStoreInputValidator {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(FederationMembershipStateStoreInputValidator.class);
+
+ private FederationMembershipStateStoreInputValidator() {
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link SubClusterRegisterRequest} for
+ * registration a new subcluster is valid or not.
+ *
+ * @param request the {@link SubClusterRegisterRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSubClusterRegisterRequest(
+ SubClusterRegisterRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing SubClusterRegister Request."
+ + " Please try again by specifying a"
+ + " SubCluster Register Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+
+ }
+
+ // validate subcluster info
+ checkSubClusterInfo(request.getSubClusterInfo());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link SubClusterDeregisterRequest} for
+ * deregistration a subcluster is valid or not.
+ *
+ * @param request the {@link SubClusterDeregisterRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSubClusterDeregisterRequest(
+ SubClusterDeregisterRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing SubClusterDeregister Request."
+ + " Please try again by specifying a"
+ + " SubCluster Deregister Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(request.getSubClusterId());
+ // validate subcluster state
+ checkSubClusterState(request.getState());
+ if (!request.getState().isFinal()) {
+ String message = "Invalid non-final state: " + request.getState();
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link SubClusterHeartbeatRequest} for
+ * heartbeating a subcluster is valid or not.
+ *
+ * @param request the {@link SubClusterHeartbeatRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSubClusterHeartbeatRequest(
+ SubClusterHeartbeatRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing SubClusterHeartbeat Request."
+ + " Please try again by specifying a"
+ + " SubCluster Heartbeat Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(request.getSubClusterId());
+ // validate last heartbeat timestamp
+ checkTimestamp(request.getLastHeartBeat());
+ // validate subcluster capability
+ checkCapability(request.getCapability());
+ // validate subcluster state
+ checkSubClusterState(request.getState());
+
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided {@link GetSubClusterInfoRequest} for querying
+ * subcluster's information is valid or not.
+ *
+ * @param request the {@link GetSubClusterInfoRequest} to validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateGetSubClusterInfoRequest(
+ GetSubClusterInfoRequest request)
+ throws FederationStateStoreInvalidInputException {
+
+ // check if the request is present
+ if (request == null) {
+ String message = "Missing GetSubClusterInfo Request."
+ + " Please try again by specifying a Get SubCluster information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(request.getSubClusterId());
+ }
+
+ /**
+ * Validate if the SubCluster Info are present or not.
+ *
+ * @param subClusterInfo the information of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster Info
+ * are invalid
+ */
+ private static void checkSubClusterInfo(SubClusterInfo subClusterInfo)
+ throws FederationStateStoreInvalidInputException {
+ if (subClusterInfo == null) {
+ String message = "Missing SubCluster Information."
+ + " Please try again by specifying SubCluster Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster id
+ checkSubClusterId(subClusterInfo.getSubClusterId());
+
+ // validate AMRM Service address
+ checkAddress(subClusterInfo.getAMRMServiceAddress());
+ // validate ClientRM Service address
+ checkAddress(subClusterInfo.getClientRMServiceAddress());
+ // validate RMClient Service address
+ checkAddress(subClusterInfo.getRMAdminServiceAddress());
+ // validate RMWeb Service address
+ checkAddress(subClusterInfo.getRMWebServiceAddress());
+
+ // validate last heartbeat timestamp
+ checkTimestamp(subClusterInfo.getLastHeartBeat());
+ // validate last start timestamp
+ checkTimestamp(subClusterInfo.getLastStartTime());
+
+ // validate subcluster state
+ checkSubClusterState(subClusterInfo.getState());
+
+ // validate subcluster capability
+ checkCapability(subClusterInfo.getCapability());
+ }
+
+ /**
+ * Validate if the timestamp is positive or not.
+ *
+ * @param timestamp the timestamp to be verified
+ * @throws FederationStateStoreInvalidInputException if the timestamp is
+ * invalid
+ */
+ private static void checkTimestamp(long timestamp)
+ throws FederationStateStoreInvalidInputException {
+ if (timestamp < 0) {
+ String message = "Invalid timestamp information."
+ + " Please try again by specifying valid Timestamp Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the Capability is present or not.
+ *
+ * @param capability the capability of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the capability is
+ * invalid
+ */
+ private static void checkCapability(String capability)
+ throws FederationStateStoreInvalidInputException {
+ if (capability == null || capability.isEmpty()) {
+ String message = "Invalid capability information."
+ + " Please try again by specifying valid Capability Information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the SubCluster Id is present or not.
+ *
+ * @param subClusterId the identifier of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster Id is
+ * invalid
+ */
+ protected static void checkSubClusterId(SubClusterId subClusterId)
+ throws FederationStateStoreInvalidInputException {
+ // check if cluster id is present
+ if (subClusterId == null) {
+ String message = "Missing SubCluster Id information."
+ + " Please try again by specifying Subcluster Id information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ // check if cluster id is valid
+ if (subClusterId.getId().isEmpty()) {
+ String message = "Invalid SubCluster Id information."
+ + " Please try again by specifying valid Subcluster Id.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the SubCluster Address is a valid URL or not.
+ *
+ * @param address the endpoint of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the address is invalid
+ */
+ private static void checkAddress(String address)
+ throws FederationStateStoreInvalidInputException {
+ // Ensure url is not null
+ if (address == null || address.isEmpty()) {
+ String message = "Missing SubCluster Endpoint information."
+ + " Please try again by specifying SubCluster Endpoint information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ // Validate url is well formed
+ boolean hasScheme = address.contains("://");
+ URI uri = null;
+ try {
+ uri = hasScheme ? URI.create(address)
+ : URI.create("dummyscheme://" + address);
+ } catch (IllegalArgumentException e) {
+ String message = "The provided SubCluster Endpoint does not contain a"
+ + " valid host:port authority: " + address;
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ String host = uri.getHost();
+ int port = uri.getPort();
+ String path = uri.getPath();
+ if ((host == null) || (port < 0)
+ || (!hasScheme && path != null && !path.isEmpty())) {
+ String message = "The provided SubCluster Endpoint does not contain a"
+ + " valid host:port authority: " + address;
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the SubCluster State is present or not.
+ *
+ * @param state the state of the subcluster to be verified
+ * @throws FederationStateStoreInvalidInputException if the SubCluster State
+ * is invalid
+ */
+ private static void checkSubClusterState(SubClusterState state)
+ throws FederationStateStoreInvalidInputException {
+ // check sub-cluster state is not empty
+ if (state == null) {
+ String message = "Missing SubCluster State information."
+ + " Please try again by specifying SubCluster State information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java
new file mode 100644
index 0000000..273a8ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationPolicyStoreInputValidator.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to {@code FederationPolicyStore}, allows
+ * a fail fast mechanism for invalid user inputs.
+ *
+ */
+public final class FederationPolicyStoreInputValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationPolicyStoreInputValidator.class);
+
+ private FederationPolicyStoreInputValidator() {
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided
+ * {@link GetSubClusterPolicyConfigurationRequest} for querying policy's
+ * information is valid or not.
+ *
+ * @param request the {@link GetSubClusterPolicyConfigurationRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateGetSubClusterPolicyConfigurationRequest(
+ GetSubClusterPolicyConfigurationRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing GetSubClusterPolicyConfiguration Request."
+ + " Please try again by specifying a policy selection information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate queue id
+ checkQueue(request.getQueue());
+ }
+
+ /**
+ * Quick validation on the input to check some obvious fail conditions (fail
+ * fast). Check if the provided
+ * {@link SetSubClusterPolicyConfigurationRequest} for adding a new policy is
+ * valid or not.
+ *
+ * @param request the {@link SetSubClusterPolicyConfigurationRequest} to
+ * validate against
+ * @throws FederationStateStoreInvalidInputException if the request is invalid
+ */
+ public static void validateSetSubClusterPolicyConfigurationRequest(
+ SetSubClusterPolicyConfigurationRequest request)
+ throws FederationStateStoreInvalidInputException {
+ if (request == null) {
+ String message = "Missing SetSubClusterPolicyConfiguration Request."
+ + " Please try again by specifying an policy insertion information.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate subcluster policy configuration
+ checkSubClusterPolicyConfiguration(request.getPolicyConfiguration());
+ }
+
+ /**
+ * Validate if the SubClusterPolicyConfiguration is valid or not.
+ *
+ * @param policyConfiguration the policy information to be verified
+ * @throws FederationStateStoreInvalidInputException if the policy information
+ * are invalid
+ */
+ private static void checkSubClusterPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration)
+ throws FederationStateStoreInvalidInputException {
+ if (policyConfiguration == null) {
+ String message = "Missing SubClusterPolicyConfiguration."
+ + " Please try again by specifying a SubClusterPolicyConfiguration.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+
+ // validate queue id
+ checkQueue(policyConfiguration.getQueue());
+ // validate policy type
+ checkType(policyConfiguration.getType());
+
+ }
+
+ /**
+ * Validate if the queue id is a valid or not.
+ *
+ * @param queue the queue id of the policy to be verified
+ * @throws FederationStateStoreInvalidInputException if the queue id is
+ * invalid
+ */
+ private static void checkQueue(String queue)
+ throws FederationStateStoreInvalidInputException {
+ if (queue == null || queue.isEmpty()) {
+ String message = "Missing Queue. Please try again by specifying a Queue.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+ /**
+ * Validate if the policy type is a valid or not.
+ *
+ * @param type the type of the policy to be verified
+ * @throws FederationStateStoreInvalidInputException if the policy is invalid
+ */
+ private static void checkType(String type)
+ throws FederationStateStoreInvalidInputException {
+ if (type == null || type.isEmpty()) {
+ String message = "Missing Policy Type."
+ + " Please try again by specifying a Policy Type.";
+ LOG.warn(message);
+ throw new FederationStateStoreInvalidInputException(message);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java
new file mode 100644
index 0000000..ea1428d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreInvalidInputException.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Exception thrown by the {@link FederationMembershipStateStoreInputValidator},
+ * {@link FederationApplicationHomeSubClusterStoreInputValidator},
+ * {@link FederationPolicyStoreInputValidator} if the input is invalid.
+ *
+ */
+public class FederationStateStoreInvalidInputException extends YarnException {
+
+ /**
+ * IDE auto-generated.
+ */
+ private static final long serialVersionUID = -7352144682711430801L;
+
+ public FederationStateStoreInvalidInputException(Throwable cause) {
+ super(cause);
+ }
+
+ public FederationStateStoreInvalidInputException(String message) {
+ super(message);
+ }
+
+ public FederationStateStoreInvalidInputException(String message,
+ Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java
new file mode 100644
index 0000000..f4a9c7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.utils;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d57781/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 414696b..63a5b65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -162,9 +162,9 @@ public abstract class FederationStateStoreBaseTest {
SubClusterRegisterRequest.newInstance(subClusterInfo2));
stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
- .newInstance(subClusterId1, SubClusterState.SC_RUNNING, ""));
- stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
- .newInstance(subClusterId2, SubClusterState.SC_UNHEALTHY, ""));
+ .newInstance(subClusterId1, SubClusterState.SC_RUNNING, "capability"));
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest.newInstance(
+ subClusterId2, SubClusterState.SC_UNHEALTHY, "capability"));
Assert.assertTrue(
stateStore.getSubClusters(GetSubClustersInfoRequest.newInstance(true))
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-5300. Exclude generated
federation protobuf sources from YARN Javadoc/findbugs build
Posted by su...@apache.org.
YARN-5300. Exclude generated federation protobuf sources from YARN Javadoc/findbugs build
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5235a684
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5235a684
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5235a684
Branch: refs/heads/YARN-2915
Commit: 5235a6847979d1adf8179c4ab8fd681abbf455cf
Parents: 53d372a
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Jul 19 15:08:25 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml | 3 +++
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5235a684/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c090749..2f5451d 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -21,6 +21,9 @@
<Package name="org.apache.hadoop.yarn.proto" />
</Match>
<Match>
+ <Package name="org.apache.hadoop.yarn.federation.proto" />
+ </Match>
+ <Match>
<Class name="~org\.apache\.hadoop\.yarn\.ipc\.RpcProtos.*" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5235a684/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index c43588a..99b8b5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
- <excludePackageNames>org.apache.hadoop.yarn.proto</excludePackageNames>
+ <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto</excludePackageNames>
</configuration>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-3672. Create Facade for
Federation State and Policy Store. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-3672. Create Facade for Federation State and Policy Store. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c73ed1ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c73ed1ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c73ed1ff
Branch: refs/heads/YARN-2915
Commit: c73ed1ff00ca232546d982a59be1d4a52be594ce
Parents: dd9ed27
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 17 11:13:19 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 13 +
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../src/main/resources/yarn-default.xml | 20 +-
.../hadoop-yarn-server-common/pom.xml | 10 +
.../utils/FederationStateStoreFacade.java | 532 +++++++++++++++++++
.../server/federation/utils/package-info.java | 17 +
.../utils/FederationStateStoreTestUtil.java | 149 ++++++
.../utils/TestFederationStateStoreFacade.java | 148 ++++++
9 files changed, 905 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c8aa857..fa73134 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -93,6 +93,9 @@
<apacheds.version>2.0.0-M21</apacheds.version>
<ldap-api.version>1.0.0-M33</ldap-api.version>
+ <jcache.version>1.0.0</jcache.version>
+ <ehcache.version>3.0.3</ehcache.version>
+
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -1240,6 +1243,16 @@
<artifactId>kerb-simplekdc</artifactId>
<version>1.0.0-RC2</version>
</dependency>
+ <dependency>
+ <groupId>javax.cache</groupId>
+ <artifactId>cache-api</artifactId>
+ <version>${jcache.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>${ehcache.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8901797..d27b4e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2489,6 +2489,19 @@ public class YarnConfiguration extends Configuration {
////////////////////////////////
public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+
+ public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
+ FEDERATION_PREFIX + "state-store.class";
+
+ public static final String DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS =
+ "org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore";
+
+ public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
+ FEDERATION_PREFIX + "cache-ttl.secs";
+
+ // 5 minutes
+ public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3da4bab..bfc2534 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -68,6 +68,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
+ // Federation default configs to be ignored
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
+
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
configurationPropsToSkipCompare.add(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 09c4309..37e4f40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2613,8 +2613,8 @@
<description>The arguments to pass to the Node label script.</description>
<name>yarn.nodemanager.node-labels.provider.script.opts</name>
</property>
- <!-- Other Configuration -->
+ <!-- Federation Configuration -->
<property>
<description>
Machine list file to be loaded by the FederationSubCluster Resolver
@@ -2623,6 +2623,24 @@
</property>
<property>
+ <description>
+ Store class name for federation state store
+ </description>
+ <name>yarn.federation.state-store.class</name>
+ <value>org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore</value>
+ </property>
+
+ <property>
+ <description>
+ The time in seconds after which the federation state store local cache
+ will be refreshed periodically
+ </description>
+ <name>yarn.federation.cache-ttl.secs</name>
+ <value>300</value>
+ </property>
+
+ <!-- Other Configuration -->
+ <property>
<description>The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 6d2fbef..cfb2533 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -108,6 +108,16 @@
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
+ <dependency>
+ <groupId>javax.cache</groupId>
+ <artifactId>cache-api</artifactId>
+ <version>${jcache.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <version>${ehcache.version}</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
new file mode 100644
index 0000000..f1c8218
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -0,0 +1,532 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import javax.cache.Cache;
+import javax.cache.CacheManager;
+import javax.cache.Caching;
+import javax.cache.configuration.CompleteConfiguration;
+import javax.cache.configuration.FactoryBuilder;
+import javax.cache.configuration.MutableConfiguration;
+import javax.cache.expiry.CreatedExpiryPolicy;
+import javax.cache.expiry.Duration;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.integration.CacheLoader;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.spi.CachingProvider;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ *
+ * The FederationStateStoreFacade is an utility wrapper that provides singleton
+ * access to the Federation state store. It abstracts out retries and in
+ * addition, it also implements the caching for various objects.
+ *
+ */
+public final class FederationStateStoreFacade {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreFacade.class);
+
+ private static final String GET_SUBCLUSTERS_CACHEID = "getSubClusters";
+ private static final String GET_POLICIES_CONFIGURATIONS_CACHEID =
+ "getPoliciesConfigurations";
+
+ private static final FederationStateStoreFacade FACADE =
+ new FederationStateStoreFacade();
+
+ private FederationStateStore stateStore;
+ private int cacheTimeToLive;
+ private Configuration conf;
+ private Cache<Object, Object> cache;
+
+ private FederationStateStoreFacade() {
+ initializeFacadeInternal(new Configuration());
+ }
+
+ private void initializeFacadeInternal(Configuration config) {
+ this.conf = config;
+ try {
+ this.stateStore = (FederationStateStore) createRetryInstance(this.conf,
+ YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS,
+ FederationStateStore.class, createRetryPolicy(conf));
+ this.stateStore.init(conf);
+
+ initCache();
+
+ } catch (YarnException ex) {
+ LOG.error("Failed to initialize the FederationStateStoreFacade object",
+ ex);
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ * Delete and re-initialize the cache, to force it to use the given
+ * configuration.
+ *
+ * @param store the {@link FederationStateStore} instance to reinitialize with
+ * @param config the updated configuration to reinitialize with
+ */
+ @VisibleForTesting
+ public synchronized void reinitialize(FederationStateStore store,
+ Configuration config) {
+ this.conf = config;
+ this.stateStore = store;
+ clearCache();
+ initCache();
+ }
+
+ public static RetryPolicy createRetryPolicy(Configuration conf) {
+ // Retry settings for StateStore
+ RetryPolicy retryPolicy = RetryPolicies.exponentialBackoffRetry(
+ conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, Integer.SIZE),
+ conf.getLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
+ YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS),
+ TimeUnit.MILLISECONDS);
+
+ return retryPolicy;
+ }
+
+ private boolean isCachingEnabled() {
+ return (cacheTimeToLive > 0);
+ }
+
+ private void initCache() {
+ // Picking the JCache provider from classpath, need to make sure there's
+ // no conflict or pick up a specific one in the future
+ cacheTimeToLive =
+ conf.getInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS,
+ YarnConfiguration.DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS);
+ if (isCachingEnabled()) {
+ CachingProvider jcacheProvider = Caching.getCachingProvider();
+ CacheManager jcacheManager = jcacheProvider.getCacheManager();
+ this.cache = jcacheManager.getCache(this.getClass().getSimpleName());
+ if (this.cache == null) {
+ LOG.info("Creating a JCache Manager with name "
+ + this.getClass().getSimpleName());
+ Duration cacheExpiry = new Duration(TimeUnit.SECONDS, cacheTimeToLive);
+ CompleteConfiguration<Object, Object> configuration =
+ new MutableConfiguration<Object, Object>().setStoreByValue(false)
+ .setReadThrough(true)
+ .setExpiryPolicyFactory(
+ new FactoryBuilder.SingletonFactory<ExpiryPolicy>(
+ new CreatedExpiryPolicy(cacheExpiry)))
+ .setCacheLoaderFactory(
+ new FactoryBuilder.SingletonFactory<CacheLoader<Object, Object>>(
+ new CacheLoaderImpl<Object, Object>()));
+ this.cache = jcacheManager.createCache(this.getClass().getSimpleName(),
+ configuration);
+ }
+ }
+ }
+
+ private void clearCache() {
+ CachingProvider jcacheProvider = Caching.getCachingProvider();
+ CacheManager jcacheManager = jcacheProvider.getCacheManager();
+
+ jcacheManager.destroyCache(this.getClass().getSimpleName());
+ this.cache = null;
+ }
+
+ /**
+ * Returns the singleton instance of the FederationStateStoreFacade object.
+ *
+ * @return the singleton {@link FederationStateStoreFacade} instance
+ */
+ public static FederationStateStoreFacade getInstance() {
+ return FACADE;
+ }
+
+ /**
+ * Returns the {@link SubClusterInfo} for the specified {@link SubClusterId}.
+ *
+ * @param subClusterId the identifier of the sub-cluster
+ * @return the sub cluster information
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterInfo getSubCluster(final SubClusterId subClusterId)
+ throws YarnException {
+ if (isCachingEnabled()) {
+ return getSubClusters(false).get(subClusterId);
+ } else {
+ return stateStore
+ .getSubCluster(GetSubClusterInfoRequest.newInstance(subClusterId))
+ .getSubClusterInfo();
+ }
+ }
+
+ /**
+ * Updates the cache with the central {@link FederationStateStore} and returns
+ * the {@link SubClusterInfo} for the specified {@link SubClusterId}.
+ *
+ * @param subClusterId the identifier of the sub-cluster
+ * @param flushCache flag to indicate if the cache should be flushed or not
+ * @return the sub cluster information
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterInfo getSubCluster(final SubClusterId subClusterId,
+ final boolean flushCache) throws YarnException {
+ if (flushCache && isCachingEnabled()) {
+ LOG.info("Flushing subClusters from cache and rehydrating from store,"
+ + " most likely on account of RM failover.");
+ cache.remove(buildGetSubClustersCacheRequest(false));
+ }
+ return getSubCluster(subClusterId);
+ }
+
+ /**
+ * Returns the {@link SubClusterInfo} of all active sub cluster(s).
+ *
+ * @param filterInactiveSubClusters whether to filter out inactive
+ * sub-clusters
+ * @return the information of all active sub cluster(s)
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ @SuppressWarnings("unchecked")
+ public Map<SubClusterId, SubClusterInfo> getSubClusters(
+ final boolean filterInactiveSubClusters) throws YarnException {
+ try {
+ if (isCachingEnabled()) {
+ return (Map<SubClusterId, SubClusterInfo>) cache
+ .get(buildGetSubClustersCacheRequest(filterInactiveSubClusters));
+ } else {
+ return buildSubClusterInfoMap(stateStore.getSubClusters(
+ GetSubClustersInfoRequest.newInstance(filterInactiveSubClusters)));
+ }
+ } catch (Throwable ex) {
+ throw new YarnException(ex);
+ }
+ }
+
+ /**
+ * Returns the {@link SubClusterPolicyConfiguration} for the specified queue.
+ *
+ * @param queue the queue whose policy is required
+ * @return the corresponding configured policy
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterPolicyConfiguration getPolicyConfiguration(
+ final String queue) throws YarnException {
+ if (isCachingEnabled()) {
+ return getPoliciesConfigurations().get(queue);
+ } else {
+ return stateStore
+ .getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue))
+ .getPolicyConfiguration();
+ }
+
+ }
+
+ /**
+ * Get the policies that is represented as
+ * {@link SubClusterPolicyConfiguration} for all currently active queues in
+ * the system.
+ *
+ * @return the policies for all currently active queues in the system
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ @SuppressWarnings("unchecked")
+ public Map<String, SubClusterPolicyConfiguration> getPoliciesConfigurations()
+ throws YarnException {
+ try {
+ if (isCachingEnabled()) {
+ return (Map<String, SubClusterPolicyConfiguration>) cache
+ .get(buildGetPoliciesConfigurationsCacheRequest());
+ } else {
+ return buildPolicyConfigMap(stateStore.getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest.newInstance()));
+ }
+ } catch (Throwable ex) {
+ throw new YarnException(ex);
+ }
+ }
+
+ /**
+ * Adds the home {@link SubClusterId} for the specified {@link ApplicationId}.
+ *
+ * @param appHomeSubCluster the mapping of the application to it's home
+ * sub-cluster
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public void addApplicationHomeSubCluster(
+ ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
+ stateStore.addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+ return;
+ }
+
+ /**
+ * Updates the home {@link SubClusterId} for the specified
+ * {@link ApplicationId}.
+ *
+ * @param appHomeSubCluster the mapping of the application to it's home
+ * sub-cluster
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public void updateApplicationHomeSubCluster(
+ ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
+ stateStore.updateApplicationHomeSubCluster(
+ UpdateApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+ return;
+ }
+
+ /**
+ * Returns the home {@link SubClusterId} for the specified
+ * {@link ApplicationId}.
+ *
+ * @param appId the identifier of the application
+ * @return the home sub cluster identifier
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public SubClusterId getApplicationHomeSubCluster(ApplicationId appId)
+ throws YarnException {
+ GetApplicationHomeSubClusterResponse response =
+ stateStore.getApplicationHomeSubCluster(
+ GetApplicationHomeSubClusterRequest.newInstance(appId));
+ return response.getApplicationHomeSubCluster().getHomeSubCluster();
+ }
+
+ /**
+ * Helper method to create instances of Object using the class name defined in
+ * the configuration object. The instances creates {@link RetryProxy} using
+ * the specific {@link RetryPolicy}.
+ *
+ * @param conf the yarn configuration
+ * @param configuredClassName the configuration provider key
+ * @param defaultValue the default implementation for fallback
+ * @param type the class for which a retry proxy is required
+ * @param retryPolicy the policy for retrying method call failures
+ * @return a retry proxy for the specified interface
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> Object createRetryInstance(Configuration conf,
+ String configuredClassName, String defaultValue, Class<T> type,
+ RetryPolicy retryPolicy) {
+
+ String className = conf.get(configuredClassName, defaultValue);
+ try {
+ Class<?> clusterResolverClass = conf.getClassByName(className);
+ if (type.isAssignableFrom(clusterResolverClass)) {
+ return RetryProxy.create(type,
+ (T) ReflectionUtils.newInstance(clusterResolverClass, conf),
+ retryPolicy);
+ } else {
+ throw new YarnRuntimeException(
+ "Class: " + className + " not instance of " + type.getSimpleName());
+ }
+ } catch (Exception e) {
+ throw new YarnRuntimeException("Could not instantiate : " + className, e);
+ }
+ }
+
+ private Map<SubClusterId, SubClusterInfo> buildSubClusterInfoMap(
+ final GetSubClustersInfoResponse response) {
+ List<SubClusterInfo> subClusters = response.getSubClusters();
+ Map<SubClusterId, SubClusterInfo> subClustersMap =
+ new HashMap<>(subClusters.size());
+ for (SubClusterInfo subCluster : subClusters) {
+ subClustersMap.put(subCluster.getSubClusterId(), subCluster);
+ }
+ return subClustersMap;
+ }
+
+ private Object buildGetSubClustersCacheRequest(
+ final boolean filterInactiveSubClusters) {
+ final String cacheKey = buildCacheKey(getClass().getSimpleName(),
+ GET_SUBCLUSTERS_CACHEID, null);
+ CacheRequest<String, Map<SubClusterId, SubClusterInfo>> cacheRequest =
+ new CacheRequest<String, Map<SubClusterId, SubClusterInfo>>(cacheKey,
+ new Func<String, Map<SubClusterId, SubClusterInfo>>() {
+ @Override
+ public Map<SubClusterId, SubClusterInfo> invoke(String key)
+ throws Exception {
+ GetSubClustersInfoResponse subClusters =
+ stateStore.getSubClusters(GetSubClustersInfoRequest
+ .newInstance(filterInactiveSubClusters));
+ return buildSubClusterInfoMap(subClusters);
+ }
+ });
+ return cacheRequest;
+ }
+
+ private Map<String, SubClusterPolicyConfiguration> buildPolicyConfigMap(
+ GetSubClusterPoliciesConfigurationsResponse response) {
+ List<SubClusterPolicyConfiguration> policyConfigs =
+ response.getPoliciesConfigs();
+ Map<String, SubClusterPolicyConfiguration> queuePolicyConfigs =
+ new HashMap<>();
+ for (SubClusterPolicyConfiguration policyConfig : policyConfigs) {
+ queuePolicyConfigs.put(policyConfig.getQueue(), policyConfig);
+ }
+ return queuePolicyConfigs;
+ }
+
+ private Object buildGetPoliciesConfigurationsCacheRequest() {
+ final String cacheKey = buildCacheKey(getClass().getSimpleName(),
+ GET_POLICIES_CONFIGURATIONS_CACHEID, null);
+ CacheRequest<String, Map<String, SubClusterPolicyConfiguration>> cacheRequest =
+ new CacheRequest<String, Map<String, SubClusterPolicyConfiguration>>(
+ cacheKey,
+ new Func<String, Map<String, SubClusterPolicyConfiguration>>() {
+ @Override
+ public Map<String, SubClusterPolicyConfiguration> invoke(
+ String key) throws Exception {
+ GetSubClusterPoliciesConfigurationsResponse policyConfigs =
+ stateStore.getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest
+ .newInstance());
+ return buildPolicyConfigMap(policyConfigs);
+ }
+ });
+ return cacheRequest;
+ }
+
+ protected String buildCacheKey(String typeName, String methodName,
+ String argName) {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append(typeName).append(".");
+ buffer.append(methodName);
+ if (argName != null) {
+ buffer.append("::");
+ buffer.append(argName);
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Internal class that implements the CacheLoader interface that can be
+ * plugged into the CacheManager to load objects into the cache for specified
+ * keys.
+ */
+ private static class CacheLoaderImpl<K, V> implements CacheLoader<K, V> {
+ @SuppressWarnings("unchecked")
+ @Override
+ public V load(K key) throws CacheLoaderException {
+ try {
+ CacheRequest<K, V> query = (CacheRequest<K, V>) key;
+ assert query != null;
+ return query.getValue();
+ } catch (Throwable ex) {
+ throw new CacheLoaderException(ex);
+ }
+ }
+
+ @Override
+ public Map<K, V> loadAll(Iterable<? extends K> keys)
+ throws CacheLoaderException {
+ // The FACADE does not use the Cache's getAll API. Hence this is not
+ // required to be implemented
+ throw new NotImplementedException();
+ }
+ }
+
+ /**
+ * Internal class that encapsulates the cache key and a function that returns
+ * the value for the specified key.
+ */
+ private static class CacheRequest<K, V> {
+ private K key;
+ private Func<K, V> func;
+
+ public CacheRequest(K key, Func<K, V> func) {
+ this.key = key;
+ this.func = func;
+ }
+
+ public V getValue() throws Exception {
+ return func.invoke(key);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((key == null) ? 0 : key.hashCode());
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ CacheRequest<K, V> other = (CacheRequest<K, V>) obj;
+ if (key == null) {
+ if (other.key != null) {
+ return false;
+ }
+ } else if (!key.equals(other.key)) {
+ return false;
+ }
+
+ return true;
+ }
+ }
+
+ /**
+ * Encapsulates a method that has one parameter and returns a value of the
+ * type specified by the TResult parameter.
+ */
+ protected interface Func<T, TResult> {
+ TResult invoke(T input) throws Exception;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java
new file mode 100644
index 0000000..39a46ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.utils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
new file mode 100644
index 0000000..c179521
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+/**
+ * Utility class for FederationStateStore unit tests.
+ */
+public class FederationStateStoreTestUtil {
+
+ private static final MonotonicClock CLOCK = new MonotonicClock();
+
+ public static final String SC_PREFIX = "SC-";
+ public static final String Q_PREFIX = "queue-";
+ public static final String POLICY_PREFIX = "policy-";
+
+ private FederationStateStore stateStore;
+
+ public FederationStateStoreTestUtil(FederationStateStore stateStore) {
+ this.stateStore = stateStore;
+ }
+
+ private SubClusterInfo createSubClusterInfo(SubClusterId subClusterId) {
+
+ String amRMAddress = "1.2.3.4:1";
+ String clientRMAddress = "1.2.3.4:2";
+ String rmAdminAddress = "1.2.3.4:3";
+ String webAppAddress = "1.2.3.4:4";
+
+ return SubClusterInfo.newInstance(subClusterId, amRMAddress,
+ clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
+ CLOCK.getTime(), "capability");
+ }
+
+ private void registerSubCluster(SubClusterId subClusterId)
+ throws YarnException {
+
+ SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ }
+
+ public void registerSubClusters(int numSubClusters) throws YarnException {
+
+ for (int i = 0; i < numSubClusters; i++) {
+ registerSubCluster(SubClusterId.newInstance(SC_PREFIX + i));
+ }
+ }
+
+ private void addApplicationHomeSC(ApplicationId appId,
+ SubClusterId subClusterId) throws YarnException {
+ ApplicationHomeSubCluster ahsc =
+ ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+ AddApplicationHomeSubClusterRequest request =
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc);
+ stateStore.addApplicationHomeSubCluster(request);
+ }
+
+ public void addAppsHomeSC(long clusterTs, int numApps) throws YarnException {
+ for (int i = 0; i < numApps; i++) {
+ addApplicationHomeSC(ApplicationId.newInstance(clusterTs, i),
+ SubClusterId.newInstance(SC_PREFIX + i));
+ }
+ }
+
+ private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
+ String policyType) {
+ return SubClusterPolicyConfiguration.newInstance(queueName, policyType,
+ ByteBuffer.allocate(1));
+ }
+
+ private void setPolicyConf(String queue, String policyType)
+ throws YarnException {
+ SetSubClusterPolicyConfigurationRequest request =
+ SetSubClusterPolicyConfigurationRequest
+ .newInstance(createSCPolicyConf(queue, policyType));
+ stateStore.setPolicyConfiguration(request);
+ }
+
+ public void addPolicyConfigs(int numQueues) throws YarnException {
+
+ for (int i = 0; i < numQueues; i++) {
+ setPolicyConf(Q_PREFIX + i, POLICY_PREFIX + i);
+ }
+ }
+
+ public SubClusterInfo querySubClusterInfo(SubClusterId subClusterId)
+ throws YarnException {
+ GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+ return stateStore.getSubCluster(request).getSubClusterInfo();
+ }
+
+ public SubClusterId queryApplicationHomeSC(ApplicationId appId)
+ throws YarnException {
+ GetApplicationHomeSubClusterRequest request =
+ GetApplicationHomeSubClusterRequest.newInstance(appId);
+
+ GetApplicationHomeSubClusterResponse response =
+ stateStore.getApplicationHomeSubCluster(request);
+
+ return response.getApplicationHomeSubCluster().getHomeSubCluster();
+ }
+
+ public SubClusterPolicyConfiguration queryPolicyConfiguration(String queue)
+ throws YarnException {
+ GetSubClusterPolicyConfigurationRequest request =
+ GetSubClusterPolicyConfigurationRequest.newInstance(queue);
+
+ GetSubClusterPolicyConfigurationResponse result =
+ stateStore.getPolicyConfiguration(request);
+ return result.getPolicyConfiguration();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c73ed1ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
new file mode 100644
index 0000000..53f4f84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Unit tests for FederationStateStoreFacade.
+ */
+@RunWith(Parameterized.class)
+public class TestFederationStateStoreFacade {
+
+ @Parameters
+ public static Collection<Boolean[]> getParameters() {
+ return Arrays
+ .asList(new Boolean[][] {{Boolean.FALSE }, {Boolean.TRUE } });
+ }
+
+ private final long clusterTs = System.currentTimeMillis();
+ private final int numSubClusters = 3;
+ private final int numApps = 5;
+ private final int numQueues = 2;
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private FederationStateStoreTestUtil stateStoreTestUtil;
+ private FederationStateStoreFacade facade =
+ FederationStateStoreFacade.getInstance();
+
+ public TestFederationStateStoreFacade(Boolean isCachingEnabled) {
+ conf = new Configuration();
+ if (!(isCachingEnabled.booleanValue())) {
+ conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
+ }
+ }
+
+ @Before
+ public void setUp() throws IOException, YarnException {
+ stateStore = new MemoryFederationStateStore();
+ stateStore.init(conf);
+ facade.reinitialize(stateStore, conf);
+ // hydrate the store
+ stateStoreTestUtil = new FederationStateStoreTestUtil(stateStore);
+ stateStoreTestUtil.registerSubClusters(numSubClusters);
+ stateStoreTestUtil.addAppsHomeSC(clusterTs, numApps);
+ stateStoreTestUtil.addPolicyConfigs(numQueues);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stateStore.close();
+ stateStore = null;
+ }
+
+ @Test
+ public void testGetSubCluster() throws YarnException {
+ for (int i = 0; i < numSubClusters; i++) {
+ SubClusterId subClusterId =
+ SubClusterId.newInstance(FederationStateStoreTestUtil.SC_PREFIX + i);
+ Assert.assertEquals(stateStoreTestUtil.querySubClusterInfo(subClusterId),
+ facade.getSubCluster(subClusterId));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterFlushCache() throws YarnException {
+ for (int i = 0; i < numSubClusters; i++) {
+ SubClusterId subClusterId =
+ SubClusterId.newInstance(FederationStateStoreTestUtil.SC_PREFIX + i);
+ Assert.assertEquals(stateStoreTestUtil.querySubClusterInfo(subClusterId),
+ facade.getSubCluster(subClusterId, true));
+ }
+ }
+
+ @Test
+ public void testGetSubClusters() throws YarnException {
+ Map<SubClusterId, SubClusterInfo> subClusters =
+ facade.getSubClusters(false);
+ for (SubClusterId subClusterId : subClusters.keySet()) {
+ Assert.assertEquals(stateStoreTestUtil.querySubClusterInfo(subClusterId),
+ subClusters.get(subClusterId));
+ }
+ }
+
+ @Test
+ public void testGetPolicyConfiguration() throws YarnException {
+ for (int i = 0; i < numQueues; i++) {
+ String queue = FederationStateStoreTestUtil.Q_PREFIX + i;
+ Assert.assertEquals(stateStoreTestUtil.queryPolicyConfiguration(queue),
+ facade.getPolicyConfiguration(queue));
+ }
+ }
+
+ @Test
+ public void testGetPoliciesConfigurations() throws YarnException {
+ Map<String, SubClusterPolicyConfiguration> queuePolicies =
+ facade.getPoliciesConfigurations();
+ for (String queue : queuePolicies.keySet()) {
+ Assert.assertEquals(stateStoreTestUtil.queryPolicyConfiguration(queue),
+ queuePolicies.get(queue));
+ }
+ }
+
+ @Test
+ public void testGetHomeSubClusterForApp() throws YarnException {
+ for (int i = 0; i < numApps; i++) {
+ ApplicationId appId = ApplicationId.newInstance(clusterTs, i);
+ Assert.assertEquals(stateStoreTestUtil.queryApplicationHomeSC(appId),
+ facade.getApplicationHomeSubCluster(appId));
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDFS-11438. Fix typo in error
message of StoragePolicyAdmin tool. Contributed by Alison Yu.
Posted by su...@apache.org.
HDFS-11438. Fix typo in error message of StoragePolicyAdmin tool. Contributed by Alison Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d150f061
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d150f061
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d150f061
Branch: refs/heads/YARN-2915
Commit: d150f061f4ebde923fda28ea898a9606b8789758
Parents: 0013090
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Feb 22 15:16:09 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Feb 22 15:16:09 2017 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d150f061/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index 4e4f018..f0643b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -259,7 +259,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
final String path = StringUtils.popOptionWithArgument("-path", args);
if (path == null) {
System.err.println("Please specify the path from which "
- + "the storage policy will be unsetd.\nUsage: " + getLongUsage());
+ + "the storage policy will be unset.\nUsage: " + getLongUsage());
return 1;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5676. Add a
HashBasedRouterPolicy,
and small policies and test refactoring. (Carlo Curino via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
index 4975a9f..5fa02d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.PriorityBroadcastPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
deleted file mode 100644
index 542a5ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
-import org.junit.Before;
-
-/**
- * Simple test of {@link UniformBroadcastPolicyManager}.
- */
-public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
-
- @Before
- public void setup() {
- //config policy
- wfp = new UniformBroadcastPolicyManager();
- wfp.setQueue("queue1");
-
- //set expected params that the base test class will use for tests
- expectedPolicyManager = UniformBroadcastPolicyManager.class;
- expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
- expectedRouterPolicy = UniformRandomRouterPolicy.class;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
deleted file mode 100644
index ab9cec4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Simple test of {@link WeightedLocalityPolicyManager}.
- */
-public class TestWeightedLocalityPolicyManager extends
- BasePolicyManagerTest {
-
- private WeightedPolicyInfo policyInfo;
-
- @Before
- public void setup() {
- // configure a policy
-
- wfp = new WeightedLocalityPolicyManager();
- wfp.setQueue("queue1");
- SubClusterId sc1 = SubClusterId.newInstance("sc1");
- SubClusterId sc2 = SubClusterId.newInstance("sc2");
- policyInfo = new WeightedPolicyInfo();
-
- Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
- routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
- routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
- policyInfo.setRouterPolicyWeights(routerWeights);
-
- Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
- amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f);
- amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f);
- policyInfo.setAMRMPolicyWeights(amrmWeights);
-
- ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo(
- policyInfo);
-
- //set expected params that the base test class will use for tests
- expectedPolicyManager = WeightedLocalityPolicyManager.class;
- expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class;
- expectedRouterPolicy = WeightedRandomRouterPolicy.class;
- }
-
- @Test
- public void testPolicyInfoSetCorrectly() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy,
- expectedRouterPolicy);
-
- //check the policyInfo propagates through ser/der correctly
- Assert.assertEquals(((WeightedLocalityPolicyManager) wfp)
- .getWeightedPolicyInfo(), policyInfo);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java
new file mode 100644
index 0000000..3cf73b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/BasePolicyManagerTest.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class provides common test methods for testing {@code
+ * FederationPolicyManager}s.
+ */
+public abstract class BasePolicyManagerTest {
+
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected FederationPolicyManager wfp = null;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedPolicyManager;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedAMRMProxyPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class expectedRouterPolicy;
+
+ @Test
+ public void testSerializeAndInstantiate() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+ }
+
+ @Test(expected = FederationPolicyInitializationException.class)
+ public void testSerializeAndInstantiateBad1() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, String.class,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad2() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ String.class, expectedRouterPolicy);
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testSerializeAndInstantiateBad3() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, String.class);
+ }
+
+ protected static void serializeAndDeserializePolicyManager(
+ FederationPolicyManager wfp, Class policyManagerType,
+ Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception {
+
+ // serializeConf it in a context
+ SubClusterPolicyConfiguration fpc = wfp.serializeConf();
+ fpc.setType(policyManagerType.getCanonicalName());
+ FederationPolicyInitializationContext context =
+ new FederationPolicyInitializationContext();
+ context.setSubClusterPolicyConfiguration(fpc);
+ context
+ .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
+ context.setFederationSubclusterResolver(
+ FederationPoliciesTestUtil.initResolver());
+ context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster"));
+
+ // based on the "context" created instantiate new class and use it
+ Class c = Class.forName(wfp.getClass().getCanonicalName());
+ FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance();
+
+ FederationAMRMProxyPolicy federationAMRMProxyPolicy =
+ wfp2.getAMRMPolicy(context, null);
+
+ // needed only for tests (getARMRMPolicy change the "type" in conf)
+ fpc.setType(wfp.getClass().getCanonicalName());
+
+ FederationRouterPolicy federationRouterPolicy =
+ wfp2.getRouterPolicy(context, null);
+
+ Assert.assertEquals(federationAMRMProxyPolicy.getClass(),
+ expAMRMProxyPolicy);
+
+ Assert.assertEquals(federationRouterPolicy.getClass(), expRouterPolicy);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java
new file mode 100644
index 0000000..5fc4a56
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestHashBasedBroadcastPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.HashBasedRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link HashBroadcastPolicyManager}.
+ */
+public class TestHashBasedBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ // config policy
+ wfp = new HashBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = HashBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = HashBasedRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..21b39e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestPriorityBroadcastPolicyManager.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test of {@link PriorityBroadcastPolicyManager}.
+ */
+public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new PriorityBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo);
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = PriorityBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = PriorityRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy, expectedRouterPolicy);
+
+ // check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(
+ ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(),
+ policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..57fafdc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestUniformBroadcastPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link UniformBroadcastPolicyManager}.
+ */
+public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ //config policy
+ wfp = new UniformBroadcastPolicyManager();
+ wfp.setQueue("queue1");
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = UniformBroadcastPolicyManager.class;
+ expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
+ expectedRouterPolicy = UniformRandomRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..51661473
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestWeightedLocalityPolicyManager.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Simple test of {@link WeightedLocalityPolicyManager}.
+ */
+public class TestWeightedLocalityPolicyManager extends
+ BasePolicyManagerTest {
+
+ private WeightedPolicyInfo policyInfo;
+
+ @Before
+ public void setup() {
+ // configure a policy
+
+ wfp = new WeightedLocalityPolicyManager();
+ wfp.setQueue("queue1");
+ SubClusterId sc1 = SubClusterId.newInstance("sc1");
+ SubClusterId sc2 = SubClusterId.newInstance("sc2");
+ policyInfo = new WeightedPolicyInfo();
+
+ Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
+ routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setRouterPolicyWeights(routerWeights);
+
+ Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
+ amrmWeights.put(new SubClusterIdInfo(sc1), 0.2f);
+ amrmWeights.put(new SubClusterIdInfo(sc2), 0.8f);
+ policyInfo.setAMRMPolicyWeights(amrmWeights);
+
+ ((WeightedLocalityPolicyManager) wfp).setWeightedPolicyInfo(
+ policyInfo);
+
+ //set expected params that the base test class will use for tests
+ expectedPolicyManager = WeightedLocalityPolicyManager.class;
+ expectedAMRMProxyPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ expectedRouterPolicy = WeightedRandomRouterPolicy.class;
+ }
+
+ @Test
+ public void testPolicyInfoSetCorrectly() throws Exception {
+ serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
+ expectedAMRMProxyPolicy,
+ expectedRouterPolicy);
+
+ //check the policyInfo propagates through ser/der correctly
+ Assert.assertEquals(((WeightedLocalityPolicyManager) wfp)
+ .getWeightedPolicyInfo(), policyInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
new file mode 100644
index 0000000..2e7a0af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Base class for router policies tests, tests for null input cases.
+ */
+public abstract class BaseRouterPoliciesTest
+ extends BaseFederationPoliciesTest {
+
+ @Test
+ public void testNullQueueRouting() throws YarnException {
+ FederationRouterPolicy localPolicy = (FederationRouterPolicy) getPolicy();
+ ApplicationSubmissionContext applicationSubmissionContext =
+ ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+ false, false, 0, Resources.none(), null, false, null, null);
+ SubClusterId chosen =
+ localPolicy.getHomeSubcluster(applicationSubmissionContext);
+ Assert.assertNotNull(chosen);
+ }
+
+ @Test(expected = FederationPolicyException.class)
+ public void testNullAppContext() throws YarnException {
+ ((FederationRouterPolicy) getPolicy()).getHomeSubcluster(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
new file mode 100644
index 0000000..af7fe43
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link HashBasedRouterPolicy}. Tests that one of
+ * the active sub-cluster is chosen.
+ */
+public class TestHashBasedRouterPolicy extends BaseRouterPoliciesTest {
+
+ private int numSubclusters = 10;
+
+ @Before
+ public void setUp() throws Exception {
+
+ // set policy in base class
+ setPolicy(new HashBasedRouterPolicy());
+
+ // setting up the active sub-clusters for this test
+ setMockActiveSubclusters(numSubclusters);
+
+ // initialize policy with context
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+ }
+
+ @Test
+ public void testHashSpreadUniformlyAmongSubclusters() throws YarnException {
+ SubClusterId chosen;
+
+ Map<SubClusterId, AtomicLong> counter = new HashMap<>();
+ for (SubClusterId id : getActiveSubclusters().keySet()) {
+ counter.put(id, new AtomicLong(0));
+ }
+
+ long jobPerSub = 100;
+
+ ApplicationSubmissionContext applicationSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ for (int i = 0; i < jobPerSub * numSubclusters; i++) {
+ when(applicationSubmissionContext.getQueue()).thenReturn("queue" + i);
+ chosen = ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(applicationSubmissionContext);
+ counter.get(chosen).addAndGet(1);
+ }
+
+ // hash spread the jobs equally among the subclusters
+ for (AtomicLong a : counter.values()) {
+ Assert.assertEquals(a.get(), jobPerSub);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index 906e35f..b70b4aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -21,7 +21,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -36,7 +35,7 @@ import org.junit.Test;
* Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load
* is properly considered for allocation.
*/
-public class TestLoadBasedRouterPolicy extends BaseFederationPoliciesTest {
+public class TestLoadBasedRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
index eefcfd9..42d919d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -23,7 +23,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -38,7 +37,7 @@ import org.junit.Test;
* Simple test class for the {@link PriorityRouterPolicy}. Tests that the
* weights are correctly used for ordering the choice of sub-clusters.
*/
-public class TestPriorityRouterPolicy extends BaseFederationPoliciesTest {
+public class TestPriorityRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
index ac41ab5..b45aa2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
@@ -21,7 +21,6 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -36,7 +35,7 @@ import org.junit.Test;
* Simple test class for the {@link UniformRandomRouterPolicy}. Tests that one
* of the active subcluster is chosen.
*/
-public class TestUniformRandomRouterPolicy extends BaseFederationPoliciesTest {
+public class TestUniformRandomRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
index 78967d0..34cc5f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -24,8 +24,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -41,7 +41,7 @@ import org.junit.Test;
* number of randomized tests to check we are weighiting correctly even if
* clusters go inactive.
*/
-public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
+public class TestWeightedRandomRouterPolicy extends BaseRouterPoliciesTest {
@Before
public void setUp() throws Exception {
@@ -78,13 +78,18 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
@Test
public void testClusterChosenWithRightProbability() throws YarnException {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ when(context.getQueue()).thenReturn("queue1");
+ setApplicationSubmissionContext(context);
+
Map<SubClusterId, AtomicLong> counter = new HashMap<>();
for (SubClusterIdInfo id : getPolicyInfo().getRouterPolicyWeights()
.keySet()) {
counter.put(id.toId(), new AtomicLong(0));
}
- float numberOfDraws = 1000000;
+ float numberOfDraws = 100000;
for (float i = 0; i < numberOfDraws; i++) {
SubClusterId chosenId = ((FederationRouterPolicy) getPolicy())
@@ -113,8 +118,8 @@ public class TestWeightedRandomRouterPolicy extends BaseFederationPoliciesTest {
Assert.assertTrue(
"Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ " expected weight: " + expectedWeight,
- expectedWeight == 0 || (actualWeight / expectedWeight) < 1.1
- && (actualWeight / expectedWeight) > 0.9);
+ expectedWeight == 0 || (actualWeight / expectedWeight) < 1.2
+ && (actualWeight / expectedWeight) > 0.8);
} else {
Assert
.assertTrue(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-3673. Create a FailoverProxy
for Federation services. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-3673. Create a FailoverProxy for Federation services. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/258bd9a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/258bd9a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/258bd9a9
Branch: refs/heads/YARN-2915
Commit: 258bd9a9bff95ac7c7f0d19593be0304a207014f
Parents: 23d5778
Author: Jian He <ji...@apache.org>
Authored: Mon Aug 22 14:43:07 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/conf/HAUtil.java | 30 ++-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../TestFederationRMFailoverProxyProvider.java | 154 ++++++++++++++
.../hadoop/yarn/client/ClientRMProxy.java | 4 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 23 +-
.../src/main/resources/yarn-default.xml | 7 +
.../hadoop-yarn-server-common/pom.xml | 2 -
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +-
.../failover/FederationProxyProviderUtil.java | 163 ++++++++++++++
.../FederationRMFailoverProxyProvider.java | 211 +++++++++++++++++++
.../federation/failover/package-info.java | 17 ++
12 files changed, 613 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
index e4948e7..942b08a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.conf;
-import com.google.common.annotations.VisibleForTesting;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -27,8 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.Private
public class HAUtil {
@@ -44,6 +45,29 @@ public class HAUtil {
}
/**
+ * Returns true if Federation is configured.
+ *
+ * @param conf Configuration
+ * @return true if federation is configured in the configuration; else false.
+ */
+ public static boolean isFederationEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ }
+
+ /**
+ * Returns true if RM failover is enabled in a Federation setting.
+ *
+ * @param conf Configuration
+ * @return if RM failover is enabled in conjunction with Federation in the
+ * configuration; else false.
+ */
+ public static boolean isFederationFailoverEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+ }
+
+ /**
* Returns true if Resource Manager HA is configured.
*
* @param conf Configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d27b4e5..29176d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2490,6 +2490,16 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+ public static final String FEDERATION_ENABLED = FEDERATION_PREFIX + "enabled";
+ public static final boolean DEFAULT_FEDERATION_ENABLED = false;
+
+ public static final String FEDERATION_FAILOVER_ENABLED =
+ FEDERATION_PREFIX + "failover.enabled";
+ public static final boolean DEFAULT_FEDERATION_FAILOVER_ENABLED = true;
+
+ public static final String FEDERATION_SUBCLUSTER_ID =
+ FEDERATION_PREFIX + "sub-cluster.id";
+
public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
FEDERATION_PREFIX + "state-store.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index bfc2534..c4d8f38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -71,6 +71,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Federation default configs to be ignored
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
new file mode 100644
index 0000000..fa3523c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for FederationRMFailoverProxyProvider.
+ */
+public class TestFederationRMFailoverProxyProvider {
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private final String dummyCapability = "cap";
+
+ @Before
+ public void setUp() throws IOException, YarnException {
+ conf = new YarnConfiguration();
+ stateStore = new MemoryFederationStateStore();
+ stateStore.init(conf);
+ FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stateStore.close();
+ stateStore = null;
+ }
+
+ @Test
+ public void testFederationRMFailoverProxyProvider() throws Exception {
+ final SubClusterId subClusterId = SubClusterId.newInstance("SC-1");
+ final MiniYARNCluster cluster = new MiniYARNCluster(
+ "testFederationRMFailoverProxyProvider", 3, 0, 1, 1);
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
+
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ 2000);
+
+ HATestUtil.setRpcAddressForRM("rm1", 10000, conf);
+ HATestUtil.setRpcAddressForRM("rm2", 20000, conf);
+ HATestUtil.setRpcAddressForRM("rm3", 30000, conf);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+
+ cluster.init(conf);
+ cluster.start();
+
+ // Transition rm3 to active;
+ makeRMActive(subClusterId, cluster, 2);
+
+ ApplicationClientProtocol client = FederationProxyProviderUtil
+ .createRMProxy(conf, ApplicationClientProtocol.class, subClusterId,
+ UserGroupInformation.getCurrentUser());
+
+ // client will retry until the rm becomes active.
+ GetClusterMetricsResponse response =
+ client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+
+ // validate response
+ checkResponse(response);
+
+ // transition rm3 to standby
+ cluster.getResourceManager(2).getRMContext().getRMAdminService()
+ .transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER));
+
+ // Transition rm2 to active;
+ makeRMActive(subClusterId, cluster, 1);
+ response = client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+
+ // validate response
+ checkResponse(response);
+
+ cluster.stop();
+ }
+
+ private void checkResponse(GetClusterMetricsResponse response) {
+ Assert.assertNotNull(response.getClusterMetrics());
+ Assert.assertEquals(0,
+ response.getClusterMetrics().getNumActiveNodeManagers());
+ }
+
+ private void makeRMActive(final SubClusterId subClusterId,
+ final MiniYARNCluster cluster, final int index) {
+ try {
+ System.out.println("Transition rm" + (index + 1) + " to active");
+ String dummyAddress = "host:" + index;
+ cluster.getResourceManager(index).getRMContext().getRMAdminService()
+ .transitionToActive(new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER));
+ ResourceManager rm = cluster.getResourceManager(index);
+ InetSocketAddress amRMAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ InetSocketAddress clientRMAddress =
+ rm.getClientRMService().getBindAddress();
+ SubClusterRegisterRequest request = SubClusterRegisterRequest
+ .newInstance(SubClusterInfo.newInstance(subClusterId,
+ amRMAddress.getAddress().getHostAddress() + ":"
+ + amRMAddress.getPort(),
+ clientRMAddress.getAddress().getHostAddress() + ":"
+ + clientRMAddress.getPort(),
+ dummyAddress, dummyAddress, SubClusterState.SC_NEW, 1,
+ dummyCapability));
+ stateStore.registerSubCluster(request);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
index b29263e..6365662 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
@@ -84,7 +84,7 @@ public class ClientRMProxy<T> extends RMProxy<T> {
@Private
@Override
- protected InetSocketAddress getRMAddress(YarnConfiguration conf,
+ public InetSocketAddress getRMAddress(YarnConfiguration conf,
Class<?> protocol) throws IOException {
if (protocol == ApplicationClientProtocol.class) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
@@ -111,7 +111,7 @@ public class ClientRMProxy<T> extends RMProxy<T> {
@Private
@Override
- protected void checkAllowedProtocols(Class<?> protocol) {
+ public void checkAllowedProtocols(Class<?> protocol) {
Preconditions.checkArgument(
protocol.isAssignableFrom(ClientRMProtocols.class),
"RM does not support this client protocol");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index 3ab06bd..f93a182 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -33,8 +33,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -64,14 +64,14 @@ public class RMProxy<T> {
* Verify the passed protocol is supported.
*/
@Private
- protected void checkAllowedProtocols(Class<?> protocol) {}
+ public void checkAllowedProtocols(Class<?> protocol) {}
/**
* Get the ResourceManager address from the provided Configuration for the
* given protocol.
*/
@Private
- protected InetSocketAddress getRMAddress(
+ public InetSocketAddress getRMAddress(
YarnConfiguration conf, Class<?> protocol) throws IOException {
throw new UnsupportedOperationException("This method should be invoked " +
"from an instance of ClientRMProxy or ServerRMProxy");
@@ -90,7 +90,8 @@ public class RMProxy<T> {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
: new YarnConfiguration(configuration);
- RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
+ RetryPolicy retryPolicy = createRetryPolicy(conf,
+ (HAUtil.isHAEnabled(conf) || HAUtil.isFederationFailoverEnabled(conf)));
return newProxyInstance(conf, protocol, instance, retryPolicy);
}
@@ -116,7 +117,7 @@ public class RMProxy<T> {
private static <T> T newProxyInstance(final YarnConfiguration conf,
final Class<T> protocol, RMProxy instance, RetryPolicy retryPolicy)
throws IOException{
- if (HAUtil.isHAEnabled(conf)) {
+ if (HAUtil.isHAEnabled(conf) || HAUtil.isFederationEnabled(conf)) {
RMFailoverProxyProvider<T> provider =
instance.createRMFailoverProxyProvider(conf, protocol);
return (T) RetryProxy.create(protocol, provider, retryPolicy);
@@ -146,7 +147,8 @@ public class RMProxy<T> {
@Deprecated
public static <T> T createRMProxy(final Configuration conf,
final Class<T> protocol, InetSocketAddress rmAddress) throws IOException {
- RetryPolicy retryPolicy = createRetryPolicy(conf, HAUtil.isHAEnabled(conf));
+ RetryPolicy retryPolicy = createRetryPolicy(conf,
+ (HAUtil.isHAEnabled(conf) || HAUtil.isFederationFailoverEnabled(conf)));
T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
LOG.info("Connecting to ResourceManager at " + rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
@@ -155,9 +157,16 @@ public class RMProxy<T> {
/**
* Get a proxy to the RM at the specified address. To be used to create a
* RetryProxy.
+ *
+ * @param conf Configuration to generate retry policy
+ * @param protocol Protocol for the proxy
+ * @param rmAddress Address of the ResourceManager
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException on failure
*/
@Private
- static <T> T getProxy(final Configuration conf,
+ public static <T> T getProxy(final Configuration conf,
final Class<T> protocol, final InetSocketAddress rmAddress)
throws IOException {
return UserGroupInformation.getCurrentUser().doAs(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 37e4f40..93dcbe6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2617,6 +2617,13 @@
<!-- Federation Configuration -->
<property>
<description>
+ Flag to indicate whether the RM is participating in Federation or not.
+ </description>
+ <name>yarn.federation.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <description>
Machine list file to be loaded by the FederationSubCluster Resolver
</description>
<name>yarn.federation.machine-list</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index cfb2533..5ae8889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -111,12 +111,10 @@
<dependency>
<groupId>javax.cache</groupId>
<artifactId>cache-api</artifactId>
- <version>${jcache.version}</version>
</dependency>
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>ehcache</artifactId>
- <version>${ehcache.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index 8555fc3..b3038e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -71,7 +71,7 @@ public class ServerRMProxy<T> extends RMProxy<T> {
@InterfaceAudience.Private
@Override
- protected InetSocketAddress getRMAddress(YarnConfiguration conf,
+ public InetSocketAddress getRMAddress(YarnConfiguration conf,
Class<?> protocol) {
if (protocol == ResourceTracker.class) {
return conf.getSocketAddr(
@@ -93,7 +93,7 @@ public class ServerRMProxy<T> extends RMProxy<T> {
@InterfaceAudience.Private
@Override
- protected void checkAllowedProtocols(Class<?> protocol) {
+ public void checkAllowedProtocols(Class<?> protocol) {
Preconditions.checkArgument(
protocol.isAssignableFrom(ResourceTracker.class),
"ResourceManager does not support this protocol");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
new file mode 100644
index 0000000..a986008
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.failover;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class that creates proxy for specified protocols when federation is
+ * enabled. The class creates a federation aware failover provider, i.e. the
+ * failover provider uses the {@code FederationStateStore} to determine the
+ * current active ResourceManager
+ */
+@Private
+@Unstable
+public final class FederationProxyProviderUtil {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationProxyProviderUtil.class);
+
+ /**
+ * Create a proxy for the specified protocol. For non-HA, this is a direct
+ * connection to the ResourceManager address. When HA is enabled, the proxy
+ * handles the failover between the ResourceManagers as well.
+ *
+ * @param configuration Configuration to generate {@link ClientRMProxy}
+ * @param protocol Protocol for the proxy
+ * @param subClusterId the unique identifier or the sub-cluster
+ * @param user the user on whose behalf the proxy is being created
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException on failure
+ */
+ @Public
+ @Unstable
+ public static <T> T createRMProxy(Configuration configuration,
+ final Class<T> protocol, SubClusterId subClusterId,
+ UserGroupInformation user) throws IOException {
+ return createRMProxy(configuration, protocol, subClusterId, user, null);
+ }
+
+ /**
+ * Create a proxy for the specified protocol. For non-HA, this is a direct
+ * connection to the ResourceManager address. When HA is enabled, the proxy
+ * handles the failover between the ResourceManagers as well.
+ *
+ * @param configuration Configuration to generate {@link ClientRMProxy}
+ * @param protocol Protocol for the proxy
+ * @param subClusterId the unique identifier or the sub-cluster
+ * @param user the user on whose behalf the proxy is being created
+ * @param token the auth token to use for connection
+ * @param <T> Type information of the proxy
+ * @return Proxy to the RM
+ * @throws IOException on failure
+ */
+ @Public
+ @Unstable
+ @SuppressWarnings("unchecked")
+ public static <T> T createRMProxy(final Configuration configuration,
+ final Class<T> protocol, SubClusterId subClusterId,
+ UserGroupInformation user, final Token token) throws IOException {
+ try {
+ final YarnConfiguration conf = new YarnConfiguration(configuration);
+ updateConf(conf, subClusterId);
+ if (token != null) {
+ LOG.info(
+ "Creating RMProxy with a token: {} to subcluster: {}"
+ + " for protocol: {}",
+ token, subClusterId, protocol.getSimpleName());
+ user.addToken(token);
+ setAuthModeInConf(conf);
+ } else {
+ LOG.info("Creating RMProxy without a token to subcluster: {}"
+ + " for protocol: {}", subClusterId, protocol.getSimpleName());
+ }
+ final T proxyConnection = user.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws Exception {
+ return ClientRMProxy.createRMProxy(conf, protocol);
+ }
+ });
+
+ return proxyConnection;
+ } catch (IOException e) {
+ String message =
+ "Error while creating of RM application master service proxy for"
+ + " appAttemptId: " + user;
+ LOG.info(message);
+ throw new YarnRuntimeException(message, e);
+ } catch (InterruptedException e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+
+ private static void setAuthModeInConf(Configuration conf) {
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ SaslRpcServer.AuthMethod.TOKEN.toString());
+ }
+
+ // updating the conf with the refreshed RM addresses as proxy creations
+ // are based out of conf
+ private static void updateConf(Configuration conf,
+ SubClusterId subClusterId) {
+ conf.set(YarnConfiguration.FEDERATION_SUBCLUSTER_ID, subClusterId.getId());
+ // In a Federation setting, we will connect to not just the local cluster RM
+ // but also multiple external RMs. The membership information of all the RMs
+ // that are currently
+ // participating in Federation is available in the central
+ // FederationStateStore.
+ // So we will:
+ // 1. obtain the RM service addresses from FederationStateStore using the
+ // FederationRMFailoverProxyProvider.
+ // 2. disable traditional HA as that depends on local configuration lookup
+ // for RMs using indexes.
+ // 3. we will enable federation failover IF traditional HA is enabled so
+ // that the appropriate failover RetryPolicy is initialized.
+ conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+ conf.setClass(YarnConfiguration.CLIENT_FAILOVER_PROXY_PROVIDER,
+ FederationRMFailoverProxyProvider.class, RMFailoverProxyProvider.class);
+ if (HAUtil.isHAEnabled(conf)) {
+ conf.setBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+ }
+ }
+
+ // disable instantiation
+ private FederationProxyProviderUtil() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
new file mode 100644
index 0000000..90a9239
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.failover;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
+import org.apache.hadoop.yarn.client.RMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A FailoverProxyProvider implementation that uses the
+ * {@code FederationStateStore} to determine the ResourceManager to connect to.
+ * This supports both HA and regular mode which is controlled by configuration.
+ */
+@Private
+@Unstable
+public class FederationRMFailoverProxyProvider<T>
+ implements RMFailoverProxyProvider<T> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationRMFailoverProxyProvider.class);
+
+ private RMProxy<T> rmProxy;
+ private Class<T> protocol;
+ private T current;
+ private YarnConfiguration conf;
+ private FederationStateStoreFacade facade;
+ private SubClusterId subClusterId;
+ private Collection<Token<? extends TokenIdentifier>> originalTokens;
+ private boolean federationFailoverEnabled = false;
+
+ @Override
+ public void init(Configuration configuration, RMProxy<T> proxy,
+ Class<T> proto) {
+ this.rmProxy = proxy;
+ this.protocol = proto;
+ this.rmProxy.checkAllowedProtocols(this.protocol);
+ String clusterId =
+ configuration.get(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
+ Preconditions.checkNotNull(clusterId, "Missing Federation SubClusterId");
+ this.subClusterId = SubClusterId.newInstance(clusterId);
+ this.facade = facade.getInstance();
+ if (configuration instanceof YarnConfiguration) {
+ this.conf = (YarnConfiguration) configuration;
+ }
+ federationFailoverEnabled =
+ conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
+ conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES,
+ YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES));
+
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
+ conf.getInt(
+ YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS,
+ YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
+
+ try {
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ originalTokens = currentUser.getTokens();
+ LOG.info("Initialized Federation proxy for user: {}",
+ currentUser.getUserName());
+ } catch (IOException e) {
+ LOG.warn("Could not get information of requester, ignoring for now.");
+ }
+
+ }
+
+ private void addOriginalTokens(UserGroupInformation currentUser) {
+ if (originalTokens == null || originalTokens.isEmpty()) {
+ return;
+ }
+ for (Token<? extends TokenIdentifier> token : originalTokens) {
+ currentUser.addToken(token);
+ }
+ }
+
+ private T getProxyInternal(boolean isFailover) {
+ SubClusterInfo subClusterInfo;
+ UserGroupInformation currentUser = null;
+ try {
+ LOG.info("Failing over to the ResourceManager for SubClusterId: {}",
+ subClusterId);
+ subClusterInfo = facade.getSubCluster(subClusterId, isFailover);
+ // updating the conf with the refreshed RM addresses as proxy
+ // creations
+ // are based out of conf
+ updateRMAddress(subClusterInfo);
+ currentUser = UserGroupInformation.getCurrentUser();
+ addOriginalTokens(currentUser);
+ } catch (YarnException e) {
+ LOG.error("Exception while trying to create proxy to the ResourceManager"
+ + " for SubClusterId: {}", subClusterId, e);
+ return null;
+ } catch (IOException e) {
+ LOG.warn("Could not get information of requester, ignoring for now.");
+ }
+ try {
+ final InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
+ LOG.info("Connecting to {} with protocol {} as user: {}", rmAddress,
+ protocol.getSimpleName(), currentUser);
+ LOG.info("Failed over to the RM at {} for SubClusterId: {}", rmAddress,
+ subClusterId);
+ return RMProxy.getProxy(conf, protocol, rmAddress);
+ } catch (IOException ioe) {
+ LOG.error(
+ "IOException while trying to create proxy to the ResourceManager"
+ + " for SubClusterId: {}",
+ subClusterId, ioe);
+ return null;
+ }
+ }
+
+ private void updateRMAddress(SubClusterInfo subClusterInfo) {
+ if (subClusterInfo != null) {
+ if (protocol == ApplicationClientProtocol.class) {
+ conf.set(YarnConfiguration.RM_ADDRESS,
+ subClusterInfo.getClientRMServiceAddress());
+ } else if (protocol == ApplicationMasterProtocol.class) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ subClusterInfo.getAMRMServiceAddress());
+ } else if (protocol == ResourceManagerAdministrationProtocol.class) {
+ conf.set(YarnConfiguration.RM_ADMIN_ADDRESS,
+ subClusterInfo.getRMAdminServiceAddress());
+ }
+ }
+ }
+
+ @Override
+ public synchronized ProxyInfo<T> getProxy() {
+ if (current == null) {
+ current = getProxyInternal(false);
+ }
+ return new ProxyInfo<T>(current, subClusterId.getId());
+ }
+
+ @Override
+ public synchronized void performFailover(T currentProxy) {
+ closeInternal(currentProxy);
+ current = getProxyInternal(federationFailoverEnabled);
+ }
+
+ @Override
+ public Class<T> getInterface() {
+ return protocol;
+ }
+
+ private void closeInternal(T currentProxy) {
+ if ((currentProxy != null) && (currentProxy instanceof Closeable)) {
+ try {
+ ((Closeable) currentProxy).close();
+ } catch (IOException e) {
+ LOG.warn("Exception while trying to close proxy", e);
+ }
+ } else {
+ RPC.stopProxy(currentProxy);
+ }
+
+ }
+
+ /**
+ * Close all the proxy objects which have been opened over the lifetime of
+ * this proxy provider.
+ */
+ @Override
+ public synchronized void close() throws IOException {
+ closeInternal(current);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258bd9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java
new file mode 100644
index 0000000..b1baa0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.failover;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-6093. Minor bugs with
AMRMtoken renewal and state store availability when using
FederationRMFailoverProxyProvider during RM failover. (Botong Huang via
Subru).
Posted by su...@apache.org.
YARN-6093. Minor bugs with AMRMtoken renewal and state store availability when using FederationRMFailoverProxyProvider during RM failover. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9bada3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9bada3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9bada3e
Branch: refs/heads/YARN-2915
Commit: d9bada3eecfda07b6ae48ee149f2889d21a105ec
Parents: f4ea2bb
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Feb 22 13:16:22 2017 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../TestFederationRMFailoverProxyProvider.java | 69 +++++++++++++++
.../FederationRMFailoverProxyProvider.java | 88 +++++++++++---------
2 files changed, 118 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9bada3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
index fa3523c..e3f9155 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ -19,17 +19,21 @@ package org.apache.hadoop.yarn.client;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.io.retry.FailoverProxyProvider.ProxyInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
+import org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -44,6 +48,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
/**
* Unit tests for FederationRMFailoverProxyProvider.
*/
@@ -151,4 +159,65 @@ public class TestFederationRMFailoverProxyProvider {
}
}
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testUGIForProxyCreation()
+ throws IOException, InterruptedException {
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ UserGroupInformation user1 =
+ UserGroupInformation.createProxyUser("user1", currentUser);
+ UserGroupInformation user2 =
+ UserGroupInformation.createProxyUser("user2", currentUser);
+
+ final TestableFederationRMFailoverProxyProvider provider =
+ new TestableFederationRMFailoverProxyProvider();
+
+ InetSocketAddress addr =
+ conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ final ClientRMProxy rmProxy = mock(ClientRMProxy.class);
+ when(rmProxy.getRMAddress(any(YarnConfiguration.class), any(Class.class)))
+ .thenReturn(addr);
+
+ user1.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() {
+ provider.init(conf, rmProxy, ApplicationMasterProtocol.class);
+ return null;
+ }
+ });
+
+ final ProxyInfo currentProxy = provider.getProxy();
+ Assert.assertEquals("user1", provider.getLastProxyUGI().getUserName());
+
+ user2.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() {
+ provider.performFailover(currentProxy.proxy);
+ return null;
+ }
+ });
+ Assert.assertEquals("user1", provider.getLastProxyUGI().getUserName());
+
+ provider.close();
+ }
+
+ protected static class TestableFederationRMFailoverProxyProvider<T>
+ extends FederationRMFailoverProxyProvider<T> {
+
+ private UserGroupInformation lastProxyUGI = null;
+
+ @Override
+ protected T createRMProxy(InetSocketAddress rmAddress) throws IOException {
+ lastProxyUGI = UserGroupInformation.getCurrentUser();
+ return super.createRMProxy(rmAddress);
+ }
+
+ public UserGroupInformation getLastProxyUGI() {
+ return lastProxyUGI;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9bada3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index 0ffab0b..7c21882 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.federation.failover;
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.Collection;
+import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -29,14 +29,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.client.RMFailoverProxyProvider;
import org.apache.hadoop.yarn.client.RMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -44,6 +42,7 @@ import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/**
@@ -64,7 +63,7 @@ public class FederationRMFailoverProxyProvider<T>
private YarnConfiguration conf;
private FederationStateStoreFacade facade;
private SubClusterId subClusterId;
- private Collection<Token<? extends TokenIdentifier>> originalTokens;
+ private UserGroupInformation originalUser;
private boolean federationFailoverEnabled = false;
@Override
@@ -97,59 +96,67 @@ public class FederationRMFailoverProxyProvider<T>
YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
try {
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- originalTokens = currentUser.getTokens();
+ this.originalUser = UserGroupInformation.getCurrentUser();
LOG.info("Initialized Federation proxy for user: {}",
- currentUser.getUserName());
+ this.originalUser.getUserName());
} catch (IOException e) {
LOG.warn("Could not get information of requester, ignoring for now.");
+ this.originalUser = null;
}
}
- private void addOriginalTokens(UserGroupInformation currentUser) {
- if (originalTokens == null || originalTokens.isEmpty()) {
- return;
- }
- for (Token<? extends TokenIdentifier> token : originalTokens) {
- currentUser.addToken(token);
- }
+ @VisibleForTesting
+ protected T createRMProxy(InetSocketAddress rmAddress) throws IOException {
+ return RMProxy.getProxy(conf, protocol, rmAddress);
}
private T getProxyInternal(boolean isFailover) {
SubClusterInfo subClusterInfo;
- UserGroupInformation currentUser = null;
+ // Use the existing proxy as a backup in case getting the new proxy fails.
+ // Note that if the first time it fails, the backup is also null. In that
+ // case we will hit NullPointerException and throw it back to AM.
+ T proxy = this.current;
try {
LOG.info("Failing over to the ResourceManager for SubClusterId: {}",
subClusterId);
subClusterInfo = facade.getSubCluster(subClusterId, isFailover);
// updating the conf with the refreshed RM addresses as proxy
- // creations
- // are based out of conf
+ // creations are based out of conf
updateRMAddress(subClusterInfo);
- currentUser = UserGroupInformation.getCurrentUser();
- addOriginalTokens(currentUser);
- } catch (YarnException e) {
+ if (this.originalUser == null) {
+ InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
+ LOG.info(
+ "Connecting to {} subClusterId {} with protocol {}"
+ + " without a proxy user",
+ rmAddress, subClusterId, protocol.getSimpleName());
+ proxy = createRMProxy(rmAddress);
+ } else {
+ // If the original ugi exists, always use that to create proxy because
+ // it contains up-to-date AMRMToken
+ proxy = this.originalUser.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws IOException {
+ InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
+ LOG.info(
+ "Connecting to {} subClusterId {} with protocol {} as user {}",
+ rmAddress, subClusterId, protocol.getSimpleName(),
+ originalUser);
+ return createRMProxy(rmAddress);
+ }
+ });
+ }
+ } catch (Exception e) {
LOG.error("Exception while trying to create proxy to the ResourceManager"
+ " for SubClusterId: {}", subClusterId, e);
- return null;
- } catch (IOException e) {
- LOG.warn("Could not get information of requester, ignoring for now.");
- }
- try {
- final InetSocketAddress rmAddress = rmProxy.getRMAddress(conf, protocol);
- LOG.info("Connecting to {} with protocol {} as user: {}", rmAddress,
- protocol.getSimpleName(), currentUser);
- LOG.info("Failed over to the RM at {} for SubClusterId: {}", rmAddress,
- subClusterId);
- return RMProxy.getProxy(conf, protocol, rmAddress);
- } catch (IOException ioe) {
- LOG.error(
- "IOException while trying to create proxy to the ResourceManager"
- + " for SubClusterId: {}",
- subClusterId, ioe);
- return null;
+ if (proxy == null) {
+ throw new YarnRuntimeException(
+ String.format("Create initial proxy to the ResourceManager for"
+ + " SubClusterId %s failed", subClusterId),
+ e);
+ }
}
+ return proxy;
}
private void updateRMAddress(SubClusterInfo subClusterInfo) {
@@ -177,8 +184,11 @@ public class FederationRMFailoverProxyProvider<T>
@Override
public synchronized void performFailover(T currentProxy) {
- closeInternal(currentProxy);
+ // It will not return null proxy here
current = getProxyInternal(federationFailoverEnabled);
+ if (current != currentProxy) {
+ closeInternal(currentProxy);
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-14100. Upgrade Jsch jar to
latest version to fix vulnerability in old versions. Contributed by
Vinayakumar B
Posted by su...@apache.org.
HADOOP-14100. Upgrade Jsch jar to latest version to fix vulnerability in old versions. Contributed by Vinayakumar B
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/159d6c56
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/159d6c56
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/159d6c56
Branch: refs/heads/YARN-2915
Commit: 159d6c56e7f3aa3ebe45750cf88735287f047b42
Parents: 82607fc
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Feb 23 14:25:08 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Feb 23 14:25:08 2017 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/159d6c56/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 47e21d8..c8aa857 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -963,7 +963,7 @@
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
- <version>0.1.51</version>
+ <version>0.1.54</version>
</dependency>
<dependency>
<groupId>org.apache.htrace</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-3671. Integrate Federation
services with ResourceManager. Contributed by Subru Krishnan
Posted by su...@apache.org.
YARN-3671. Integrate Federation services with ResourceManager. Contributed by Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23e6b16c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23e6b16c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23e6b16c
Branch: refs/heads/YARN-2915
Commit: 23e6b16c361684eb8c7634173ff73029043a3d61
Parents: 258bd9a
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 30 12:20:52 2016 +0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 11 +-
.../yarn/conf/TestYarnConfigurationFields.java | 4 +-
.../failover/FederationProxyProviderUtil.java | 2 +-
.../FederationRMFailoverProxyProvider.java | 4 +-
...ationMembershipStateStoreInputValidator.java | 7 +-
.../TestFederationStateStoreInputValidator.java | 10 +-
.../server/resourcemanager/ResourceManager.java | 26 ++
.../FederationStateStoreHeartbeat.java | 108 +++++++
.../federation/FederationStateStoreService.java | 304 +++++++++++++++++++
.../federation/package-info.java | 17 ++
.../webapp/dao/ClusterMetricsInfo.java | 5 +-
.../TestFederationRMStateStoreService.java | 170 +++++++++++
12 files changed, 648 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 29176d4..7aa203d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2497,9 +2497,6 @@ public class YarnConfiguration extends Configuration {
FEDERATION_PREFIX + "failover.enabled";
public static final boolean DEFAULT_FEDERATION_FAILOVER_ENABLED = true;
- public static final String FEDERATION_SUBCLUSTER_ID =
- FEDERATION_PREFIX + "sub-cluster.id";
-
public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
FEDERATION_PREFIX + "state-store.class";
@@ -2512,6 +2509,14 @@ public class YarnConfiguration extends Configuration {
// 5 minutes
public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+ public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
+ FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
+
+ // 5 minutes
+ public static final int
+ DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
+ 5 * 60;
+
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index c4d8f38..5e0876f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -72,9 +72,9 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
configurationPropsToSkipCompare
- .add(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
- configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
index a986008..18f1338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
@@ -134,7 +134,7 @@ public final class FederationProxyProviderUtil {
// are based out of conf
private static void updateConf(Configuration conf,
SubClusterId subClusterId) {
- conf.set(YarnConfiguration.FEDERATION_SUBCLUSTER_ID, subClusterId.getId());
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
// In a Federation setting, we will connect to not just the local cluster RM
// but also multiple external RMs. The membership information of all the RMs
// that are currently
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index 90a9239..0ffab0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -74,8 +74,8 @@ public class FederationRMFailoverProxyProvider<T>
this.protocol = proto;
this.rmProxy.checkAllowedProtocols(this.protocol);
String clusterId =
- configuration.get(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
- Preconditions.checkNotNull(clusterId, "Missing Federation SubClusterId");
+ configuration.get(YarnConfiguration.RM_CLUSTER_ID);
+ Preconditions.checkNotNull(clusterId, "Missing RM ClusterId");
this.subClusterId = SubClusterId.newInstance(clusterId);
this.facade = facade.getInstance();
if (configuration instanceof YarnConfiguration) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
index b587ee5..ff9d8e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationMembershipStateStoreInputValidator.java
@@ -159,7 +159,10 @@ public final class FederationMembershipStateStoreInputValidator {
}
/**
- * Validate if the SubCluster Info are present or not.
+ * Validate if all the required fields on {@link SubClusterInfo} are present
+ * or not. {@code Capability} will be empty as the corresponding
+ * {@code ResourceManager} is in the process of initialization during
+ * registration.
*
* @param subClusterInfo the information of the subcluster to be verified
* @throws FederationStateStoreInvalidInputException if the SubCluster Info
@@ -194,8 +197,6 @@ public final class FederationMembershipStateStoreInputValidator {
// validate subcluster state
checkSubClusterState(subClusterInfo.getState());
- // validate subcluster capability
- checkCapability(subClusterInfo.getCapability());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
index 13175ae..b95f17a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
@@ -242,11 +242,8 @@ public class TestFederationStateStoreInputValidator {
SubClusterRegisterRequest.newInstance(subClusterInfo);
FederationMembershipStateStoreInputValidator
.validateSubClusterRegisterRequest(request);
- Assert.fail();
} catch (FederationStateStoreInvalidInputException e) {
- LOG.info(e.getMessage());
- Assert.assertTrue(
- e.getMessage().startsWith("Invalid capability information."));
+ Assert.fail(e.getMessage());
}
// Execution with Empty Capability
@@ -260,11 +257,8 @@ public class TestFederationStateStoreInputValidator {
SubClusterRegisterRequest.newInstance(subClusterInfo);
FederationMembershipStateStoreInputValidator
.validateSubClusterRegisterRequest(request);
- Assert.fail();
} catch (FederationStateStoreInvalidInputException e) {
- LOG.info(e.getMessage());
- Assert.assertTrue(
- e.getMessage().startsWith("Invalid capability information."));
+ Assert.fail(e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 58e4077..e4bd64e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.federation.FederationStateStoreService;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV1Publisher;
@@ -188,6 +189,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected RMAppManager rmAppManager;
protected ApplicationACLsManager applicationACLsManager;
protected QueueACLsManager queueACLsManager;
+ private FederationStateStoreService federationStateStoreService;
private WebApp webApp;
private AppReportFetcher fetcher = null;
protected ResourceTrackerService resourceTracker;
@@ -504,6 +506,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
return new RMTimelineCollectorManager(rmContext);
}
+ private FederationStateStoreService createFederationStateStoreService() {
+ return new FederationStateStoreService(rmContext);
+ }
+
protected SystemMetricsPublisher createSystemMetricsPublisher() {
SystemMetricsPublisher publisher;
if (YarnConfiguration.timelineServiceEnabled(conf) &&
@@ -733,6 +739,20 @@ public class ResourceManager extends CompositeService implements Recoverable {
delegationTokenRenewer.setRMContext(rmContext);
}
+ if(HAUtil.isFederationEnabled(conf)) {
+ String cId = YarnConfiguration.getClusterId(conf);
+ if (cId.isEmpty()) {
+ String errMsg =
+ "Cannot initialize RM as Federation is enabled"
+ + " but cluster id is not configured.";
+ LOG.error(errMsg);
+ throw new YarnRuntimeException(errMsg);
+ }
+ federationStateStoreService = createFederationStateStoreService();
+ addIfService(federationStateStoreService);
+ LOG.info("Initialized Federation membership.");
+ }
+
new RMNMInfo(rmContext, scheduler);
super.serviceInit(conf);
@@ -1397,6 +1417,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
@Private
+ @VisibleForTesting
+ public FederationStateStoreService getFederationStateStoreService() {
+ return this.federationStateStoreService;
+ }
+
+ @Private
WebApp getWebapp() {
return this.webApp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java
new file mode 100644
index 0000000..a4618a2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreHeartbeat.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import java.io.StringWriter;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+
+/**
+ * Periodic heart beat from a <code>ResourceManager</code> participating in
+ * federation to indicate liveliness. The heart beat publishes the current
+ * capabilities as represented by {@link ClusterMetricsInfo} of the sub cluster.
+ *
+ */
+public class FederationStateStoreHeartbeat implements Runnable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreHeartbeat.class);
+
+ private SubClusterId subClusterId;
+ private FederationStateStore stateStoreService;
+
+ private final ResourceScheduler rs;
+
+ private StringWriter currentClusterState;
+ private JSONJAXBContext jc;
+ private JSONMarshaller marshaller;
+ private String capability;
+
+ public FederationStateStoreHeartbeat(SubClusterId subClusterId,
+ FederationStateStore stateStoreClient, ResourceScheduler scheduler) {
+ this.stateStoreService = stateStoreClient;
+ this.subClusterId = subClusterId;
+ this.rs = scheduler;
+ // Initialize the JAXB Marshaller
+ this.currentClusterState = new StringWriter();
+ try {
+ this.jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
+ ClusterMetricsInfo.class);
+ marshaller = jc.createJSONMarshaller();
+ } catch (JAXBException e) {
+ LOG.warn("Exception while trying to initialize JAXB context.", e);
+ }
+ LOG.info("Initialized Federation membership for cluster with timestamp: "
+ + ResourceManager.getClusterTimeStamp());
+ }
+
+ /**
+ * Get the current cluster state as a JSON string representation of the
+ * {@link ClusterMetricsInfo}.
+ */
+ private void updateClusterState() {
+ try {
+ // get the current state
+ currentClusterState.getBuffer().setLength(0);
+ ClusterMetricsInfo clusterMetricsInfo = new ClusterMetricsInfo(rs);
+ marshaller.marshallToJSON(clusterMetricsInfo, currentClusterState);
+ capability = currentClusterState.toString();
+ } catch (Exception e) {
+ LOG.warn("Exception while trying to generate cluster state,"
+ + " so reverting to last know state.", e);
+ }
+ }
+
+ @Override
+ public synchronized void run() {
+ try {
+ updateClusterState();
+ SubClusterHeartbeatRequest request = SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, capability);
+ stateStoreService.subClusterHeartbeat(request);
+ LOG.debug("Sending the heartbeat with capability: {}", capability);
+ } catch (Exception e) {
+ LOG.warn("Exception when trying to heartbeat: ", e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
new file mode 100644
index 0000000..9a01d7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Implements {@link FederationStateStore} and provides a service for
+ * participating in the federation membership.
+ */
+public class FederationStateStoreService extends AbstractService
+ implements FederationStateStore {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreService.class);
+
+ private Configuration config;
+ private ScheduledExecutorService scheduledExecutorService;
+ private FederationStateStoreHeartbeat stateStoreHeartbeat;
+ private FederationStateStore stateStoreClient = null;
+ private SubClusterId subClusterId;
+ private long heartbeatInterval;
+ private RMContext rmContext;
+
+ public FederationStateStoreService(RMContext rmContext) {
+ super(FederationStateStoreService.class.getName());
+ LOG.info("FederationStateStoreService initialized");
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ this.config = conf;
+
+ RetryPolicy retryPolicy =
+ FederationStateStoreFacade.createRetryPolicy(conf);
+
+ this.stateStoreClient =
+ (FederationStateStore) FederationStateStoreFacade.createRetryInstance(
+ conf, YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS,
+ FederationStateStore.class, retryPolicy);
+ this.stateStoreClient.init(conf);
+ LOG.info("Initialized state store client class");
+
+ this.subClusterId =
+ SubClusterId.newInstance(YarnConfiguration.getClusterId(conf));
+
+ heartbeatInterval = conf.getLong(
+ YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+ if (heartbeatInterval <= 0) {
+ heartbeatInterval =
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS;
+ }
+ LOG.info("Initialized federation membership service.");
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ registerAndInitializeHeartbeat();
+
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ Exception ex = null;
+ try {
+ if (this.scheduledExecutorService != null
+ && !this.scheduledExecutorService.isShutdown()) {
+ this.scheduledExecutorService.shutdown();
+ LOG.info("Stopped federation membership heartbeat");
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to shutdown ScheduledExecutorService", e);
+ ex = e;
+ }
+
+ if (this.stateStoreClient != null) {
+ try {
+ deregisterSubCluster(SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED));
+ } finally {
+ this.stateStoreClient.close();
+ }
+ }
+
+ if (ex != null) {
+ throw ex;
+ }
+ }
+
+ // Return a client accessible string representation of the service address.
+ private String getServiceAddress(InetSocketAddress address) {
+ InetSocketAddress socketAddress = NetUtils.getConnectAddress(address);
+ return socketAddress.getAddress().getHostAddress() + ":"
+ + socketAddress.getPort();
+ }
+
+ private void registerAndInitializeHeartbeat() {
+ String clientRMAddress =
+ getServiceAddress(rmContext.getClientRMService().getBindAddress());
+ String amRMAddress = getServiceAddress(
+ rmContext.getApplicationMasterService().getBindAddress());
+ String rmAdminAddress = getServiceAddress(
+ config.getSocketAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
+ String webAppAddress =
+ WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(config);
+
+ SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ amRMAddress, clientRMAddress, rmAdminAddress, webAppAddress,
+ SubClusterState.SC_NEW, ResourceManager.getClusterTimeStamp(), "");
+ try {
+ registerSubCluster(SubClusterRegisterRequest.newInstance(subClusterInfo));
+ LOG.info("Successfully registered for federation subcluster: {}",
+ subClusterInfo);
+ } catch (Exception e) {
+ throw new YarnRuntimeException(
+ "Failed to register Federation membership with the StateStore", e);
+ }
+ stateStoreHeartbeat = new FederationStateStoreHeartbeat(subClusterId,
+ stateStoreClient, rmContext.getScheduler());
+ scheduledExecutorService =
+ HadoopExecutors.newSingleThreadScheduledExecutor();
+ scheduledExecutorService.scheduleWithFixedDelay(stateStoreHeartbeat,
+ heartbeatInterval, heartbeatInterval, TimeUnit.SECONDS);
+ LOG.info("Started federation membership heartbeat with interval: {}",
+ heartbeatInterval);
+ }
+
+ @VisibleForTesting
+ public FederationStateStore getStateStoreClient() {
+ return stateStoreClient;
+ }
+
+ @VisibleForTesting
+ public FederationStateStoreHeartbeat getStateStoreHeartbeatThread() {
+ return stateStoreHeartbeat;
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return stateStoreClient.getCurrentVersion();
+ }
+
+ @Override
+ public Version loadVersion() {
+ return stateStoreClient.getCurrentVersion();
+ }
+
+ @Override
+ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ return stateStoreClient.getPolicyConfiguration(request);
+ }
+
+ @Override
+ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+ return stateStoreClient.setPolicyConfiguration(request);
+ }
+
+ @Override
+ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException {
+ return stateStoreClient.getPoliciesConfigurations(request);
+ }
+
+ @Override
+ public SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest registerSubClusterRequest)
+ throws YarnException {
+ return stateStoreClient.registerSubCluster(registerSubClusterRequest);
+ }
+
+ @Override
+ public SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest subClusterDeregisterRequest)
+ throws YarnException {
+ return stateStoreClient.deregisterSubCluster(subClusterDeregisterRequest);
+ }
+
+ @Override
+ public SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest subClusterHeartbeatRequest)
+ throws YarnException {
+ return stateStoreClient.subClusterHeartbeat(subClusterHeartbeatRequest);
+ }
+
+ @Override
+ public GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest subClusterRequest) throws YarnException {
+ return stateStoreClient.getSubCluster(subClusterRequest);
+ }
+
+ @Override
+ public GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest subClustersRequest) throws YarnException {
+ return stateStoreClient.getSubClusters(subClustersRequest);
+ }
+
+ @Override
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.addApplicationHomeSubCluster(request);
+ }
+
+ @Override
+ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
+ UpdateApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.updateApplicationHomeSubCluster(request);
+ }
+
+ @Override
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
+ GetApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.getApplicationHomeSubCluster(request);
+ }
+
+ @Override
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.getApplicationsHomeSubCluster(request);
+ }
+
+ @Override
+ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
+ DeleteApplicationHomeSubClusterRequest request) throws YarnException {
+ return stateStoreClient.deleteApplicationHomeSubCluster(request);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java
new file mode 100644
index 0000000..47c7c65
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index f083b05..dc42eb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -65,7 +65,10 @@ public class ClusterMetricsInfo {
} // JAXB needs this
public ClusterMetricsInfo(final ResourceManager rm) {
- ResourceScheduler rs = rm.getResourceScheduler();
+ this(rm.getResourceScheduler());
+ }
+
+ public ClusterMetricsInfo(final ResourceScheduler rs) {
QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23e6b16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
new file mode 100644
index 0000000..30f69b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+
+/**
+ * Unit tests for FederationStateStoreService.
+ */
+public class TestFederationRMStateStoreService {
+
+ private final HAServiceProtocol.StateChangeRequestInfo requestInfo =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ private final SubClusterId subClusterId = SubClusterId.newInstance("SC-1");
+ private final GetSubClusterInfoRequest request =
+ GetSubClusterInfoRequest.newInstance(subClusterId);
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private long lastHearbeatTS = 0;
+ private JSONJAXBContext jc;
+ private JSONUnmarshaller unmarshaller;
+
+ @Before
+ public void setUp() throws IOException, YarnException, JAXBException {
+ conf = new YarnConfiguration();
+ jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
+ ClusterMetricsInfo.class);
+ unmarshaller = jc.createJSONUnmarshaller();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ unmarshaller = null;
+ jc = null;
+ }
+
+ @Test
+ public void testFederationStateStoreService() throws Exception {
+ conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
+ final MockRM rm = new MockRM(conf);
+
+ // Initially there should be no entry for the sub-cluster
+ rm.init(conf);
+ stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+ try {
+ stateStore.getSubCluster(request);
+ Assert.fail("There should be no entry for the sub-cluster.");
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().endsWith("does not exist"));
+ }
+
+ // Validate if sub-cluster is registered
+ rm.start();
+ String capability = checkSubClusterInfo(SubClusterState.SC_NEW);
+ Assert.assertTrue(capability.isEmpty());
+
+ // Heartbeat to see if sub-cluster transitions to running
+ FederationStateStoreHeartbeat storeHeartbeat =
+ rm.getFederationStateStoreService().getStateStoreHeartbeatThread();
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 0);
+
+ // heartbeat again after adding a node.
+ rm.registerNode("127.0.0.1:1234", 4 * 1024);
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 1);
+
+ // Validate sub-cluster deregistration
+ rm.getFederationStateStoreService()
+ .deregisterSubCluster(SubClusterDeregisterRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED));
+ checkSubClusterInfo(SubClusterState.SC_UNREGISTERED);
+
+ // check after failover
+ explicitFailover(rm);
+
+ capability = checkSubClusterInfo(SubClusterState.SC_NEW);
+ Assert.assertTrue(capability.isEmpty());
+
+ // Heartbeat to see if sub-cluster transitions to running
+ storeHeartbeat =
+ rm.getFederationStateStoreService().getStateStoreHeartbeatThread();
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 0);
+
+ // heartbeat again after adding a node.
+ rm.registerNode("127.0.0.1:1234", 4 * 1024);
+ storeHeartbeat.run();
+ capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+ checkClusterMetricsInfo(capability, 1);
+
+ rm.stop();
+ }
+
+ private void explicitFailover(MockRM rm) throws IOException {
+ rm.getAdminService().transitionToStandby(requestInfo);
+ Assert.assertTrue(rm.getRMContext()
+ .getHAServiceState() == HAServiceProtocol.HAServiceState.STANDBY);
+ rm.getAdminService().transitionToActive(requestInfo);
+ Assert.assertTrue(rm.getRMContext()
+ .getHAServiceState() == HAServiceProtocol.HAServiceState.ACTIVE);
+ lastHearbeatTS = 0;
+ stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+ }
+
+ private void checkClusterMetricsInfo(String capability, int numNodes)
+ throws JAXBException {
+ ClusterMetricsInfo clusterMetricsInfo = unmarshaller.unmarshalFromJSON(
+ new StringReader(capability), ClusterMetricsInfo.class);
+ Assert.assertEquals(numNodes, clusterMetricsInfo.getTotalNodes());
+ }
+
+ private String checkSubClusterInfo(SubClusterState state)
+ throws YarnException {
+ Assert.assertNotNull(stateStore.getSubCluster(request));
+ SubClusterInfo response =
+ stateStore.getSubCluster(request).getSubClusterInfo();
+ Assert.assertEquals(state, response.getState());
+ Assert.assertTrue(response.getLastHeartBeat() >= lastHearbeatTS);
+ lastHearbeatTS = response.getLastHeartBeat();
+ return response.getCapability();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-5676. Add a
HashBasedRouterPolicy,
and small policies and test refactoring. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5676. Add a HashBasedRouterPolicy, and small policies and test refactoring. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d028a9b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d028a9b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d028a9b3
Branch: refs/heads/YARN-2915
Commit: d028a9b3b5671f082be1ee385615a4a5d3b9f8d2
Parents: 6cc9d73
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 22 15:02:22 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +-
.../policies/AbstractPolicyManager.java | 175 -----------------
.../policies/FederationPolicyManager.java | 117 ------------
.../PriorityBroadcastPolicyManager.java | 66 -------
.../federation/policies/RouterPolicyFacade.java | 1 +
.../policies/UniformBroadcastPolicyManager.java | 56 ------
.../policies/WeightedLocalityPolicyManager.java | 67 -------
.../policies/manager/AbstractPolicyManager.java | 190 +++++++++++++++++++
.../manager/FederationPolicyManager.java | 118 ++++++++++++
.../manager/HashBroadcastPolicyManager.java | 38 ++++
.../manager/PriorityBroadcastPolicyManager.java | 66 +++++++
.../manager/UniformBroadcastPolicyManager.java | 44 +++++
.../manager/WeightedLocalityPolicyManager.java | 67 +++++++
.../policies/manager/package-info.java | 19 ++
.../policies/router/AbstractRouterPolicy.java | 19 ++
.../policies/router/HashBasedRouterPolicy.java | 81 ++++++++
.../policies/router/LoadBasedRouterPolicy.java | 3 +
.../policies/router/PriorityRouterPolicy.java | 3 +
.../router/UniformRandomRouterPolicy.java | 10 +-
.../router/WeightedRandomRouterPolicy.java | 3 +
.../policies/BaseFederationPoliciesTest.java | 17 +-
.../policies/BasePolicyManagerTest.java | 108 -----------
...ionPolicyInitializationContextValidator.java | 1 +
.../TestPriorityBroadcastPolicyManager.java | 72 -------
.../policies/TestRouterPolicyFacade.java | 2 +
.../TestUniformBroadcastPolicyManager.java | 40 ----
.../TestWeightedLocalityPolicyManager.java | 79 --------
.../policies/manager/BasePolicyManagerTest.java | 104 ++++++++++
.../TestHashBasedBroadcastPolicyManager.java | 40 ++++
.../TestPriorityBroadcastPolicyManager.java | 72 +++++++
.../TestUniformBroadcastPolicyManager.java | 40 ++++
.../TestWeightedLocalityPolicyManager.java | 79 ++++++++
.../policies/router/BaseRouterPoliciesTest.java | 51 +++++
.../router/TestHashBasedRouterPolicy.java | 83 ++++++++
.../router/TestLoadBasedRouterPolicy.java | 3 +-
.../router/TestPriorityRouterPolicy.java | 3 +-
.../router/TestUniformRandomRouterPolicy.java | 3 +-
.../router/TestWeightedRandomRouterPolicy.java | 15 +-
38 files changed, 1160 insertions(+), 798 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 26d21f5..531200e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2529,7 +2529,8 @@ public class YarnConfiguration extends Configuration {
+ "policy-manager";
public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
- + ".hadoop.yarn.server.federation.policies.UniformBroadcastPolicyManager";
+ + ".hadoop.yarn.server.federation.policies"
+ + ".manager.UniformBroadcastPolicyManager";
public static final String FEDERATION_POLICY_MANAGER_PARAMS =
FEDERATION_PREFIX + "policy-manager-params";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
deleted file mode 100644
index e77f2e3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class provides basic implementation for common methods that multiple
- * policies will need to implement.
- */
-public abstract class AbstractPolicyManager implements
- FederationPolicyManager {
-
- private String queue;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class routerFederationPolicy;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class amrmProxyFederationPolicy;
-
- public static final Logger LOG =
- LoggerFactory.getLogger(AbstractPolicyManager.class);
- /**
- * This default implementation validates the
- * {@link FederationPolicyInitializationContext},
- * then checks whether it needs to reinstantiate the class (null or
- * mismatching type), and reinitialize the policy.
- *
- * @param federationPolicyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
- * instance
- *
- * @throws FederationPolicyInitializationException if the reinitalization is
- * not valid, and ensure
- * previous state is preserved
- */
- public FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext federationPolicyContext,
- FederationAMRMProxyPolicy oldInstance)
- throws FederationPolicyInitializationException {
-
- if (amrmProxyFederationPolicy == null) {
- throw new FederationPolicyInitializationException("The parameter "
- + "amrmProxyFederationPolicy should be initialized in "
- + this.getClass().getSimpleName() + " constructor.");
- }
-
- try {
- return (FederationAMRMProxyPolicy) internalPolicyGetter(
- federationPolicyContext, oldInstance, amrmProxyFederationPolicy);
- } catch (ClassCastException e) {
- throw new FederationPolicyInitializationException(e);
- }
-
- }
-
- /**
- * This default implementation validates the
- * {@link FederationPolicyInitializationContext},
- * then checks whether it needs to reinstantiate the class (null or
- * mismatching type), and reinitialize the policy.
- *
- * @param federationPolicyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return a valid and fully reinitalized {@link FederationRouterPolicy}
- * instance
- *
- * @throws FederationPolicyInitializationException if the reinitalization is
- * not valid, and ensure
- * previous state is preserved
- */
-
- public FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext federationPolicyContext,
- FederationRouterPolicy oldInstance)
- throws FederationPolicyInitializationException {
-
- //checks that sub-types properly initialize the types of policies
- if (routerFederationPolicy == null) {
- throw new FederationPolicyInitializationException("The policy "
- + "type should be initialized in " + this.getClass().getSimpleName()
- + " constructor.");
- }
-
- try {
- return (FederationRouterPolicy) internalPolicyGetter(
- federationPolicyContext, oldInstance, routerFederationPolicy);
- } catch (ClassCastException e) {
- throw new FederationPolicyInitializationException(e);
- }
- }
-
- @Override
- public String getQueue() {
- return queue;
- }
-
- @Override
- public void setQueue(String queue) {
- this.queue = queue;
- }
-
- /**
- * Common functionality to instantiate a reinitialize a {@link
- * ConfigurableFederationPolicy}.
- */
- private ConfigurableFederationPolicy internalPolicyGetter(
- final FederationPolicyInitializationContext federationPolicyContext,
- ConfigurableFederationPolicy oldInstance, Class policy)
- throws FederationPolicyInitializationException {
-
- FederationPolicyInitializationContextValidator
- .validate(federationPolicyContext, this.getClass().getCanonicalName());
-
- if (oldInstance == null || !oldInstance.getClass().equals(policy)) {
- try {
- oldInstance = (ConfigurableFederationPolicy) policy.newInstance();
- } catch (InstantiationException e) {
- throw new FederationPolicyInitializationException(e);
- } catch (IllegalAccessException e) {
- throw new FederationPolicyInitializationException(e);
- }
- }
-
- //copying the context to avoid side-effects
- FederationPolicyInitializationContext modifiedContext =
- updateContext(federationPolicyContext,
- oldInstance.getClass().getCanonicalName());
-
- oldInstance.reinitialize(modifiedContext);
- return oldInstance;
- }
-
- /**
- * This method is used to copy-on-write the context, that will be passed
- * downstream to the router/amrmproxy policies.
- */
- private FederationPolicyInitializationContext updateContext(
- FederationPolicyInitializationContext federationPolicyContext,
- String type) {
- // copying configuration and context to avoid modification of original
- SubClusterPolicyConfiguration newConf = SubClusterPolicyConfiguration
- .newInstance(federationPolicyContext
- .getSubClusterPolicyConfiguration());
- newConf.setType(type);
-
- return new FederationPolicyInitializationContext(newConf,
- federationPolicyContext.getFederationSubclusterResolver(),
- federationPolicyContext.getFederationStateStoreFacade(),
- federationPolicyContext.getHomeSubcluster());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
deleted file mode 100644
index 39fdba3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyManager.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-/**
- *
- * Implementors need to provide the ability to serliaze a policy and its
- * configuration as a {@link SubClusterPolicyConfiguration}, as well as provide
- * (re)initialization mechanics for the underlying
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
- *
- * The serialization aspects are used by admin APIs or a policy engine to store
- * a serialized configuration in the {@code FederationStateStore}, while the
- * getters methods are used to obtain a propertly inizialized policy in the
- * {@code Router} and {@code AMRMProxy} respectively.
- *
- * This interface by design binds together {@link FederationAMRMProxyPolicy} and
- * {@link FederationRouterPolicy} and provide lifecycle support for
- * serialization and deserialization, to reduce configuration mistakes
- * (combining incompatible policies).
- *
- */
-public interface FederationPolicyManager {
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
- * current context, otherwise a new instance initialized with the current
- * context is provided. If the instance is compatible with the current class
- * the implementors should attempt to reinitalize (retaining state). To affect
- * a complete policy reset oldInstance should be null.
- *
- * @param policyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return an updated {@link FederationAMRMProxyPolicy }.
- *
- * @throws FederationPolicyInitializationException if the initialization
- * cannot be completed properly. The oldInstance should be still
- * valid in case of failed initialization.
- */
- FederationAMRMProxyPolicy getAMRMPolicy(
- FederationPolicyInitializationContext policyContext,
- FederationAMRMProxyPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationRouterPolicy} reinitialized with the current
- * context, otherwise a new instance initialized with the current context is
- * provided. If the instance is compatible with the current class the
- * implementors should attempt to reinitalize (retaining state). To affect a
- * complete policy reset oldInstance shoulb be set to null.
- *
- * @param policyContext the current context
- * @param oldInstance the existing (possibly null) instance.
- *
- * @return an updated {@link FederationRouterPolicy}.
- *
- * @throws FederationPolicyInitializationException if the initalization cannot
- * be completed properly. The oldInstance should be still valid in
- * case of failed initialization.
- */
- FederationRouterPolicy getRouterPolicy(
- FederationPolicyInitializationContext policyContext,
- FederationRouterPolicy oldInstance)
- throws FederationPolicyInitializationException;
-
- /**
- * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
- * This is to be used when writing a policy object in the federation policy
- * store.
- *
- * @return a valid policy configuration representing this object
- * parametrization.
- *
- * @throws FederationPolicyInitializationException if the current state cannot
- * be serialized properly
- */
- SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException;
-
- /**
- * This method returns the queue this policy is configured for.
- *
- * @return the name of the queue.
- */
- String getQueue();
-
- /**
- * This methods provides a setter for the queue this policy is specified for.
- *
- * @param queue the name of the queue.
- */
- void setQueue(String queue);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
deleted file mode 100644
index ebdcf42..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/PriorityBroadcastPolicyManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Policy that allows operator to configure "weights" for routing. This picks a
- * {@link PriorityRouterPolicy} for the router and a
- * {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed to
- * work together.
- */
-public class PriorityBroadcastPolicyManager extends AbstractPolicyManager {
-
- private WeightedPolicyInfo weightedPolicyInfo;
-
- public PriorityBroadcastPolicyManager() {
- // this structurally hard-codes two compatible policies for Router and
- // AMRMProxy.
- routerFederationPolicy = PriorityRouterPolicy.class;
- amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
- weightedPolicyInfo = new WeightedPolicyInfo();
- }
-
- @Override
- public SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException {
- ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
- return SubClusterPolicyConfiguration.newInstance(getQueue(),
- this.getClass().getCanonicalName(), buf);
- }
-
- @VisibleForTesting
- public WeightedPolicyInfo getWeightedPolicyInfo() {
- return weightedPolicyInfo;
- }
-
- @VisibleForTesting
- public void setWeightedPolicyInfo(WeightedPolicyInfo weightedPolicyInfo) {
- this.weightedPolicyInfo = weightedPolicyInfo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
index a3fd15a..8c22623 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
deleted file mode 100644
index a01f8fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/UniformBroadcastPolicyManager.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-import java.nio.ByteBuffer;
-
-/**
- * This class represents a simple implementation of a {@code
- * FederationPolicyManager}.
- *
- * It combines the basic policies: {@link UniformRandomRouterPolicy} and
- * {@link BroadcastAMRMProxyPolicy}, which are designed to work together and
- * "spread" the load among sub-clusters uniformly.
- *
- * This simple policy might impose heavy load on the RMs and return more
- * containers than a job requested as all requests are (replicated and)
- * broadcasted.
- */
-public class UniformBroadcastPolicyManager
- extends AbstractPolicyManager {
-
- public UniformBroadcastPolicyManager() {
- //this structurally hard-codes two compatible policies for Router and
- // AMRMProxy.
- routerFederationPolicy = UniformRandomRouterPolicy.class;
- amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
- }
-
- @Override
- public SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException {
- ByteBuffer buf = ByteBuffer.allocate(0);
- return SubClusterPolicyConfiguration
- .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
deleted file mode 100644
index f3c6673..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/WeightedLocalityPolicyManager.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-
-import java.nio.ByteBuffer;
-
-/**
- * Policy that allows operator to configure "weights" for routing. This picks a
- * {@link WeightedRandomRouterPolicy} for the router and a {@link
- * LocalityMulticastAMRMProxyPolicy} for the amrmproxy as they are designed to
- * work together.
- */
-public class WeightedLocalityPolicyManager
- extends AbstractPolicyManager {
-
- private WeightedPolicyInfo weightedPolicyInfo;
-
- public WeightedLocalityPolicyManager() {
- //this structurally hard-codes two compatible policies for Router and
- // AMRMProxy.
- routerFederationPolicy = WeightedRandomRouterPolicy.class;
- amrmProxyFederationPolicy = LocalityMulticastAMRMProxyPolicy.class;
- weightedPolicyInfo = new WeightedPolicyInfo();
- }
-
- @Override
- public SubClusterPolicyConfiguration serializeConf()
- throws FederationPolicyInitializationException {
- ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
- return SubClusterPolicyConfiguration
- .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
- }
-
- @VisibleForTesting
- public WeightedPolicyInfo getWeightedPolicyInfo() {
- return weightedPolicyInfo;
- }
-
- @VisibleForTesting
- public void setWeightedPolicyInfo(
- WeightedPolicyInfo weightedPolicyInfo) {
- this.weightedPolicyInfo = weightedPolicyInfo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java
new file mode 100644
index 0000000..f7a89c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/AbstractPolicyManager.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class provides basic implementation for common methods that multiple
+ * policies will need to implement.
+ */
+public abstract class AbstractPolicyManager implements
+ FederationPolicyManager {
+
+ private String queue;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class routerFederationPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class amrmProxyFederationPolicy;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AbstractPolicyManager.class);
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ if (amrmProxyFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The parameter "
+ + "amrmProxyFederationPolicy should be initialized in "
+ + this.getClass().getSimpleName() + " constructor.");
+ }
+
+ try {
+ return (FederationAMRMProxyPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, amrmProxyFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+
+ }
+
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationRouterPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is preserved
+ */
+
+ public FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext federationPolicyContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException {
+
+ //checks that sub-types properly initialize the types of policies
+ if (routerFederationPolicy == null) {
+ throw new FederationPolicyInitializationException("The policy "
+ + "type should be initialized in " + this.getClass().getSimpleName()
+ + " constructor.");
+ }
+
+ try {
+ return (FederationRouterPolicy) internalPolicyGetter(
+ federationPolicyContext, oldInstance, routerFederationPolicy);
+ } catch (ClassCastException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ // default implementation works only for sub-classes which do not require
+ // any parameters
+ ByteBuffer buf = ByteBuffer.allocate(0);
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+
+ @Override
+ public String getQueue() {
+ return queue;
+ }
+
+ @Override
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ /**
+ * Common functionality to instantiate a reinitialize a {@link
+ * ConfigurableFederationPolicy}.
+ */
+ private ConfigurableFederationPolicy internalPolicyGetter(
+ final FederationPolicyInitializationContext federationPolicyContext,
+ ConfigurableFederationPolicy oldInstance, Class policy)
+ throws FederationPolicyInitializationException {
+
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ if (oldInstance == null || !oldInstance.getClass().equals(policy)) {
+ try {
+ oldInstance = (ConfigurableFederationPolicy) policy.newInstance();
+ } catch (InstantiationException e) {
+ throw new FederationPolicyInitializationException(e);
+ } catch (IllegalAccessException e) {
+ throw new FederationPolicyInitializationException(e);
+ }
+ }
+
+ //copying the context to avoid side-effects
+ FederationPolicyInitializationContext modifiedContext =
+ updateContext(federationPolicyContext,
+ oldInstance.getClass().getCanonicalName());
+
+ oldInstance.reinitialize(modifiedContext);
+ return oldInstance;
+ }
+
+ /**
+ * This method is used to copy-on-write the context, that will be passed
+ * downstream to the router/amrmproxy policies.
+ */
+ private FederationPolicyInitializationContext updateContext(
+ FederationPolicyInitializationContext federationPolicyContext,
+ String type) {
+ // copying configuration and context to avoid modification of original
+ SubClusterPolicyConfiguration newConf = SubClusterPolicyConfiguration
+ .newInstance(federationPolicyContext
+ .getSubClusterPolicyConfiguration());
+ newConf.setType(type);
+
+ return new FederationPolicyInitializationContext(newConf,
+ federationPolicyContext.getFederationSubclusterResolver(),
+ federationPolicyContext.getFederationStateStoreFacade(),
+ federationPolicyContext.getHomeSubcluster());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java
new file mode 100644
index 0000000..1434c80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/FederationPolicyManager.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+/**
+ *
+ * Implementors need to provide the ability to serliaze a policy and its
+ * configuration as a {@link SubClusterPolicyConfiguration}, as well as provide
+ * (re)initialization mechanics for the underlying
+ * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy}.
+ *
+ * The serialization aspects are used by admin APIs or a policy engine to store
+ * a serialized configuration in the {@code FederationStateStore}, while the
+ * getters methods are used to obtain a propertly inizialized policy in the
+ * {@code Router} and {@code AMRMProxy} respectively.
+ *
+ * This interface by design binds together {@link FederationAMRMProxyPolicy} and
+ * {@link FederationRouterPolicy} and provide lifecycle support for
+ * serialization and deserialization, to reduce configuration mistakes
+ * (combining incompatible policies).
+ *
+ */
+public interface FederationPolicyManager {
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
+ * current context, otherwise a new instance initialized with the current
+ * context is provided. If the instance is compatible with the current class
+ * the implementors should attempt to reinitalize (retaining state). To affect
+ * a complete policy reset oldInstance should be null.
+ *
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return an updated {@link FederationAMRMProxyPolicy }.
+ *
+ * @throws FederationPolicyInitializationException if the initialization
+ * cannot be completed properly. The oldInstance should be still
+ * valid in case of failed initialization.
+ */
+ FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializationContext policyContext,
+ FederationAMRMProxyPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * If the current instance is compatible, this method returns the same
+ * instance of {@link FederationRouterPolicy} reinitialized with the current
+ * context, otherwise a new instance initialized with the current context is
+ * provided. If the instance is compatible with the current class the
+ * implementors should attempt to reinitalize (retaining state). To affect a
+ * complete policy reset oldInstance shoulb be set to null.
+ *
+ * @param policyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return an updated {@link FederationRouterPolicy}.
+ *
+ * @throws FederationPolicyInitializationException if the initalization cannot
+ * be completed properly. The oldInstance should be still valid in
+ * case of failed initialization.
+ */
+ FederationRouterPolicy getRouterPolicy(
+ FederationPolicyInitializationContext policyContext,
+ FederationRouterPolicy oldInstance)
+ throws FederationPolicyInitializationException;
+
+ /**
+ * This method is invoked to derive a {@link SubClusterPolicyConfiguration}.
+ * This is to be used when writing a policy object in the federation policy
+ * store.
+ *
+ * @return a valid policy configuration representing this object
+ * parametrization.
+ *
+ * @throws FederationPolicyInitializationException if the current state cannot
+ * be serialized properly
+ */
+ SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException;
+
+ /**
+ * This method returns the queue this policy is configured for.
+ *
+ * @return the name of the queue.
+ */
+ String getQueue();
+
+ /**
+ * This methods provides a setter for the queue this policy is specified for.
+ *
+ * @param queue the name of the queue.
+ */
+ void setQueue(String queue);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java
new file mode 100644
index 0000000..08ab08f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/HashBroadcastPolicyManager.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.HashBasedRouterPolicy;
+
+/**
+ * Policy that routes applications via hashing of their queuename, and broadcast
+ * resource requests. This picks a {@link HashBasedRouterPolicy} for the router
+ * and a {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed
+ * to work together.
+ */
+public class HashBroadcastPolicyManager extends AbstractPolicyManager {
+
+ public HashBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = HashBasedRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java
new file mode 100644
index 0000000..8139e12
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/PriorityBroadcastPolicyManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link PriorityRouterPolicy} for the router and a
+ * {@link BroadcastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class PriorityBroadcastPolicyManager extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public PriorityBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = PriorityRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration.newInstance(getQueue(),
+ this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java
new file mode 100644
index 0000000..5db0466
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/UniformBroadcastPolicyManager.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
+
+/**
+ * This class represents a simple implementation of a {@code
+ * FederationPolicyManager}.
+ *
+ * It combines the basic policies: {@link UniformRandomRouterPolicy} and
+ * {@link BroadcastAMRMProxyPolicy}, which are designed to work together and
+ * "spread" the load among sub-clusters uniformly.
+ *
+ * This simple policy might impose heavy load on the RMs and return more
+ * containers than a job requested as all requests are (replicated and)
+ * broadcasted.
+ */
+public class UniformBroadcastPolicyManager extends AbstractPolicyManager {
+
+ public UniformBroadcastPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = UniformRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java
new file mode 100644
index 0000000..109b534
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/WeightedLocalityPolicyManager.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.LocalityMulticastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.WeightedRandomRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Policy that allows operator to configure "weights" for routing. This picks a
+ * {@link WeightedRandomRouterPolicy} for the router and a {@link
+ * LocalityMulticastAMRMProxyPolicy} for the amrmproxy as they are designed to
+ * work together.
+ */
+public class WeightedLocalityPolicyManager
+ extends AbstractPolicyManager {
+
+ private WeightedPolicyInfo weightedPolicyInfo;
+
+ public WeightedLocalityPolicyManager() {
+ //this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = WeightedRandomRouterPolicy.class;
+ amrmProxyFederationPolicy = LocalityMulticastAMRMProxyPolicy.class;
+ weightedPolicyInfo = new WeightedPolicyInfo();
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration serializeConf()
+ throws FederationPolicyInitializationException {
+ ByteBuffer buf = weightedPolicyInfo.toByteBuffer();
+ return SubClusterPolicyConfiguration
+ .newInstance(getQueue(), this.getClass().getCanonicalName(), buf);
+ }
+
+ @VisibleForTesting
+ public WeightedPolicyInfo getWeightedPolicyInfo() {
+ return weightedPolicyInfo;
+ }
+
+ @VisibleForTesting
+ public void setWeightedPolicyInfo(
+ WeightedPolicyInfo weightedPolicyInfo) {
+ this.weightedPolicyInfo = weightedPolicyInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java
new file mode 100644
index 0000000..9515c01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Various implementation of FederationPolicyManager. **/
+package org.apache.hadoop.yarn.server.federation.policies.manager;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
index f49af1d..730fb41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.federation.policies.router;
import java.util.Map;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -44,4 +47,20 @@ public abstract class AbstractRouterPolicy extends
}
}
+ public void validate(ApplicationSubmissionContext appSubmissionContext)
+ throws FederationPolicyException {
+
+ if (appSubmissionContext == null) {
+ throw new FederationPolicyException(
+ "Cannot route an application with null context.");
+ }
+
+ // if the queue is not specified we set it to default value, to be
+ // compatible with YARN behavior.
+ String queue = appSubmissionContext.getQueue();
+ if (queue == null) {
+ appSubmissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
new file mode 100644
index 0000000..e40e87e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+/**
+ * This {@link FederationRouterPolicy} pick a subcluster based on the hash of
+ * the job's queue name. Useful to provide a default behavior when too many
+ * queues exist in a system. This also ensures that all jobs belonging to a
+ * queue are mapped to the same sub-cluster (likely help with locality).
+ */
+public class HashBasedRouterPolicy extends AbstractRouterPolicy {
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+
+ // note: this overrides BaseRouterPolicy and ignores the weights
+ setPolicyContext(federationPolicyContext);
+ }
+
+ /**
+ * Simply picks from alphabetically-sorted active subclusters based on the
+ * hash of quey name. Jobs of the same queue will all be routed to the same
+ * sub-cluster, as far as the number of active sub-cluster and their names
+ * remain the same.
+ *
+ * @param appSubmissionContext the context for the app being submitted.
+ *
+ * @return a hash-based chosen subcluster.
+ *
+ * @throws YarnException if there are no active subclusters.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+
+ // throws if no active subclusters available
+ Map<SubClusterId, SubClusterInfo> activeSubclusters =
+ getActiveSubclusters();
+
+ validate(appSubmissionContext);
+
+ int chosenPosition = Math.abs(
+ appSubmissionContext.getQueue().hashCode() % activeSubclusters.size());
+
+ List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
+ Collections.sort(list);
+ return list.get(chosenPosition);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index 5de749f..2ca15bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -64,6 +64,9 @@ public class LoadBasedRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
index bc3a1f7..13d9140 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -36,6 +36,9 @@ public class PriorityRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
index b8f9cc3..d820449 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -48,11 +48,10 @@ public class UniformRandomRouterPolicy extends AbstractRouterPolicy {
}
@Override
- public void reinitialize(
- FederationPolicyInitializationContext policyContext)
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
- FederationPolicyInitializationContextValidator
- .validate(policyContext, this.getClass().getCanonicalName());
+ FederationPolicyInitializationContextValidator.validate(policyContext,
+ this.getClass().getCanonicalName());
// note: this overrides AbstractRouterPolicy and ignores the weights
@@ -73,6 +72,9 @@ public class UniformRandomRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
index ac75ae9..5727134 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -43,6 +43,9 @@ public class WeightedRandomRouterPolicy extends AbstractRouterPolicy {
public SubClusterId getHomeSubcluster(
ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+ // null checks and default-queue behavior
+ validate(appSubmissionContext);
+
Map<SubClusterId, SubClusterInfo> activeSubclusters =
getActiveSubclusters();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
index ba897da..6bd8bf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.federation.policies;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -35,8 +36,10 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPo
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
import org.junit.Test;
@@ -46,7 +49,7 @@ import org.junit.Test;
public abstract class BaseFederationPoliciesTest {
private ConfigurableFederationPolicy policy;
- private WeightedPolicyInfo policyInfo;
+ private WeightedPolicyInfo policyInfo = mock(WeightedPolicyInfo.class);
private Map<SubClusterId, SubClusterInfo> activeSubclusters = new HashMap<>();
private FederationPolicyInitializationContext federationPolicyContext;
private ApplicationSubmissionContext applicationSubmissionContext =
@@ -103,7 +106,7 @@ public abstract class BaseFederationPoliciesTest {
((FederationRouterPolicy) localPolicy)
.getHomeSubcluster(getApplicationSubmissionContext());
} else {
- String[] hosts = new String[] {"host1", "host2" };
+ String[] hosts = new String[] {"host1", "host2"};
List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
.createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
((FederationAMRMProxyPolicy) localPolicy)
@@ -170,4 +173,14 @@ public abstract class BaseFederationPoliciesTest {
this.homeSubCluster = homeSubCluster;
}
+ public void setMockActiveSubclusters(int numSubclusters) {
+ for (int i = 1; i <= numSubclusters; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
deleted file mode 100644
index c609886..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BasePolicyManagerTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
-import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This class provides common test methods for testing {@code
- * FederationPolicyManager}s.
- */
-public abstract class BasePolicyManagerTest {
-
-
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected FederationPolicyManager wfp = null;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class expectedPolicyManager;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class expectedAMRMProxyPolicy;
- @SuppressWarnings("checkstyle:visibilitymodifier")
- protected Class expectedRouterPolicy;
-
-
- @Test
- public void testSerializeAndInstantiate() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy,
- expectedRouterPolicy);
- }
-
- @Test(expected = FederationPolicyInitializationException.class)
- public void testSerializeAndInstantiateBad1() throws Exception {
- serializeAndDeserializePolicyManager(wfp, String.class,
- expectedAMRMProxyPolicy, expectedRouterPolicy);
- }
-
- @Test(expected = AssertionError.class)
- public void testSerializeAndInstantiateBad2() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- String.class, expectedRouterPolicy);
- }
-
- @Test(expected = AssertionError.class)
- public void testSerializeAndInstantiateBad3() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy, String.class);
- }
-
- protected static void serializeAndDeserializePolicyManager(
- FederationPolicyManager wfp, Class policyManagerType,
- Class expAMRMProxyPolicy, Class expRouterPolicy) throws Exception {
-
- // serializeConf it in a context
- SubClusterPolicyConfiguration fpc =
- wfp.serializeConf();
- fpc.setType(policyManagerType.getCanonicalName());
- FederationPolicyInitializationContext context = new
- FederationPolicyInitializationContext();
- context.setSubClusterPolicyConfiguration(fpc);
- context
- .setFederationStateStoreFacade(FederationPoliciesTestUtil.initFacade());
- context.setFederationSubclusterResolver(
- FederationPoliciesTestUtil.initResolver());
- context.setHomeSubcluster(SubClusterId.newInstance("homesubcluster"));
-
- // based on the "context" created instantiate new class and use it
- Class c = Class.forName(wfp.getClass().getCanonicalName());
- FederationPolicyManager wfp2 = (FederationPolicyManager) c.newInstance();
-
- FederationAMRMProxyPolicy federationAMRMProxyPolicy =
- wfp2.getAMRMPolicy(context, null);
-
- //needed only for tests (getARMRMPolicy change the "type" in conf)
- fpc.setType(wfp.getClass().getCanonicalName());
-
- FederationRouterPolicy federationRouterPolicy =
- wfp2.getRouterPolicy(context, null);
-
- Assert.assertEquals(federationAMRMProxyPolicy.getClass(),
- expAMRMProxyPolicy);
-
- Assert.assertEquals(federationRouterPolicy.getClass(),
- expRouterPolicy);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
index d906b92..611a486 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestFederationPolicyInitializationContextValidator.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d028a9b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
deleted file mode 100644
index 5e5bc83..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestPriorityBroadcastPolicyManager.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Simple test of {@link PriorityBroadcastPolicyManager}.
- */
-public class TestPriorityBroadcastPolicyManager extends BasePolicyManagerTest {
-
- private WeightedPolicyInfo policyInfo;
-
- @Before
- public void setup() {
- // configure a policy
-
- wfp = new PriorityBroadcastPolicyManager();
- wfp.setQueue("queue1");
- SubClusterId sc1 = SubClusterId.newInstance("sc1");
- SubClusterId sc2 = SubClusterId.newInstance("sc2");
- policyInfo = new WeightedPolicyInfo();
-
- Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
- routerWeights.put(new SubClusterIdInfo(sc1), 0.2f);
- routerWeights.put(new SubClusterIdInfo(sc2), 0.8f);
- policyInfo.setRouterPolicyWeights(routerWeights);
-
- ((PriorityBroadcastPolicyManager) wfp).setWeightedPolicyInfo(policyInfo);
-
- // set expected params that the base test class will use for tests
- expectedPolicyManager = PriorityBroadcastPolicyManager.class;
- expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
- expectedRouterPolicy = PriorityRouterPolicy.class;
- }
-
- @Test
- public void testPolicyInfoSetCorrectly() throws Exception {
- serializeAndDeserializePolicyManager(wfp, expectedPolicyManager,
- expectedAMRMProxyPolicy, expectedRouterPolicy);
-
- // check the policyInfo propagates through ser/der correctly
- Assert.assertEquals(
- ((PriorityBroadcastPolicyManager) wfp).getWeightedPolicyInfo(),
- policyInfo);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HADOOP-13817. Add a finite shell
command timeout to ShellBasedUnixGroupsMapping. (harsh)
Posted by su...@apache.org.
HADOOP-13817. Add a finite shell command timeout to ShellBasedUnixGroupsMapping. (harsh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8694deb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8694deb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8694deb
Branch: refs/heads/YARN-2915
Commit: e8694deb6ad180449f8ce6c1c8b4f84873c0587a
Parents: 50decd3
Author: Harsh J <ha...@cloudera.com>
Authored: Mon Nov 14 15:59:58 2016 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Fri Feb 24 21:34:00 2017 +0530
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 15 +++
.../security/ShellBasedUnixGroupsMapping.java | 114 +++++++++++++---
.../main/java/org/apache/hadoop/util/Shell.java | 19 ++-
.../src/main/resources/core-default.xml | 13 ++
.../hadoop/security/TestGroupsCaching.java | 19 +--
.../TestShellBasedUnixGroupsMapping.java | 135 ++++++++++++++++++-
6 files changed, 277 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index f23dd51..e1feda1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -517,6 +517,21 @@ public class CommonConfigurationKeysPublic {
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
* core-default.xml</a>
*/
+ public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS =
+ "hadoop.security.groups.shell.command.timeout";
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final long
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT =
+ 0L;
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_AUTHENTICATION =
"hadoop.security.authentication";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index 9b80be9..4146e7b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -18,17 +18,25 @@
package org.apache.hadoop.security;
import java.io.IOException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A simple shell-based implementation of {@link GroupMappingServiceProvider}
@@ -37,11 +45,28 @@ import org.apache.hadoop.util.Shell.ShellCommandExecutor;
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
@InterfaceStability.Evolving
-public class ShellBasedUnixGroupsMapping
+public class ShellBasedUnixGroupsMapping extends Configured
implements GroupMappingServiceProvider {
-
- private static final Log LOG =
- LogFactory.getLog(ShellBasedUnixGroupsMapping.class);
+
+ @VisibleForTesting
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ShellBasedUnixGroupsMapping.class);
+
+ private long timeout = 0L;
+ private static final List<String> EMPTY_GROUPS = new LinkedList<>();
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ if (conf != null) {
+ timeout = conf.getTimeDuration(
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS,
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT,
+ TimeUnit.SECONDS);
+ }
+ }
@SuppressWarnings("serial")
private static class PartialGroupNameException extends IOException {
@@ -98,7 +123,17 @@ public class ShellBasedUnixGroupsMapping
*/
protected ShellCommandExecutor createGroupExecutor(String userName) {
return new ShellCommandExecutor(
- Shell.getGroupsForUserCommand(userName), null, null, 0L);
+ getGroupsForUserCommand(userName), null, null, timeout);
+ }
+
+ /**
+ * Returns just the shell command to be used to fetch a user's groups list.
+ * This is mainly separate to make some tests easier.
+ * @param userName The username that needs to be passed into the command built
+ * @return An appropriate shell command with arguments
+ */
+ protected String[] getGroupsForUserCommand(String userName) {
+ return Shell.getGroupsForUserCommand(userName);
}
/**
@@ -109,7 +144,17 @@ public class ShellBasedUnixGroupsMapping
*/
protected ShellCommandExecutor createGroupIDExecutor(String userName) {
return new ShellCommandExecutor(
- Shell.getGroupsIDForUserCommand(userName), null, null, 0L);
+ getGroupsIDForUserCommand(userName), null, null, timeout);
+ }
+
+ /**
+ * Returns just the shell command to be used to fetch a user's group IDs list.
+ * This is mainly separate to make some tests easier.
+ * @param userName The username that needs to be passed into the command built
+ * @return An appropriate shell command with arguments
+ */
+ protected String[] getGroupsIDForUserCommand(String userName) {
+ return Shell.getGroupsIDForUserCommand(userName);
}
/**
@@ -133,8 +178,26 @@ public class ShellBasedUnixGroupsMapping
groups = resolvePartialGroupNames(user, e.getMessage(),
executor.getOutput());
} catch (PartialGroupNameException pge) {
- LOG.warn("unable to return groups for user " + user, pge);
- return new LinkedList<>();
+ LOG.warn("unable to return groups for user {}", user, pge);
+ return EMPTY_GROUPS;
+ }
+ } catch (IOException ioe) {
+ // If its a shell executor timeout, indicate so in the message
+ // but treat the result as empty instead of throwing it up,
+ // similar to how partial resolution failures are handled above
+ if (executor.isTimedOut()) {
+ LOG.warn(
+ "Unable to return groups for user '{}' as shell group lookup " +
+ "command '{}' ran longer than the configured timeout limit of " +
+ "{} seconds.",
+ user,
+ Joiner.on(' ').join(executor.getExecString()),
+ timeout
+ );
+ return EMPTY_GROUPS;
+ } else {
+ // If its not an executor timeout, we should let the caller handle it
+ throw ioe;
}
}
@@ -196,7 +259,7 @@ public class ShellBasedUnixGroupsMapping
* @param errMessage error message from the shell command
* @param groupNames the incomplete list of group names
* @return a list of resolved group names
- * @throws PartialGroupNameException
+ * @throws PartialGroupNameException if the resolution fails or times out
*/
private List<String> resolvePartialGroupNames(String userName,
String errMessage, String groupNames) throws PartialGroupNameException {
@@ -212,21 +275,29 @@ public class ShellBasedUnixGroupsMapping
throw new PartialGroupNameException("The user name '" + userName
+ "' is not found. " + errMessage);
} else {
- LOG.warn("Some group names for '" + userName + "' are not resolvable. "
- + errMessage);
+ LOG.warn("Some group names for '{}' are not resolvable. {}",
+ userName, errMessage);
// attempt to partially resolve group names
+ ShellCommandExecutor partialResolver = createGroupIDExecutor(userName);
try {
- ShellCommandExecutor exec2 = createGroupIDExecutor(userName);
- exec2.execute();
- return parsePartialGroupNames(groupNames, exec2.getOutput());
+ partialResolver.execute();
+ return parsePartialGroupNames(
+ groupNames, partialResolver.getOutput());
} catch (ExitCodeException ece) {
// If exception is thrown trying to get group id list,
// something is terribly wrong, so give up.
- throw new PartialGroupNameException("failed to get group id list for " +
- "user '" + userName + "'", ece);
+ throw new PartialGroupNameException(
+ "failed to get group id list for user '" + userName + "'", ece);
} catch (IOException ioe) {
- throw new PartialGroupNameException("can't execute the shell command to"
- + " get the list of group id for user '" + userName + "'", ioe);
+ String message =
+ "Can't execute the shell command to " +
+ "get the list of group id for user '" + userName + "'";
+ if (partialResolver.isTimedOut()) {
+ message +=
+ " because of the command taking longer than " +
+ "the configured timeout: " + timeout + " seconds";
+ }
+ throw new PartialGroupNameException(message, ioe);
}
}
}
@@ -237,7 +308,8 @@ public class ShellBasedUnixGroupsMapping
* @param groupNames a string representing the user's group names
* @return a linked list of group names
*/
- private List<String> resolveFullGroupNames(String groupNames) {
+ @VisibleForTesting
+ protected List<String> resolveFullGroupNames(String groupNames) {
StringTokenizer tokenizer =
new StringTokenizer(groupNames, Shell.TOKEN_SEPARATOR_REGEX);
List<String> groups = new LinkedList<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index ca59b0e..bfb8183 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -955,7 +955,15 @@ public abstract class Shell {
line = errReader.readLine();
}
} catch(IOException ioe) {
- LOG.warn("Error reading the error stream", ioe);
+ // Its normal to observe a "Stream closed" I/O error on
+ // command timeouts destroying the underlying process
+ // so only log a WARN if the command didn't time out
+ if (!isTimedOut()) {
+ LOG.warn("Error reading the error stream", ioe);
+ } else {
+ LOG.debug("Error reading the error stream due to shell "
+ + "command timeout", ioe);
+ }
}
}
};
@@ -1181,6 +1189,15 @@ public abstract class Shell {
}
/**
+ * Returns the timeout value set for the executor's sub-commands.
+ * @return The timeout value in seconds
+ */
+ @VisibleForTesting
+ public long getTimeoutInterval() {
+ return timeOutInterval;
+ }
+
+ /**
* Execute the shell command.
* @throws IOException if the command fails, or if the command is
* not well constructed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 1941bec..d8136ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -188,6 +188,19 @@
</property>
<property>
+ <name>hadoop.security.groups.shell.command.timeout</name>
+ <value>0s</value>
+ <description>
+ Used by the ShellBasedUnixGroupsMapping class, this property controls how
+ long to wait for the underlying shell command that is run to fetch groups.
+ Expressed in seconds (e.g. 10s, 1m, etc.), if the running command takes
+ longer than the value configured, the command is aborted and the groups
+ resolver would return a result of no groups found. A value of 0s (default)
+ would mean an infinite wait (i.e. wait until the command exits on its own).
+ </description>
+</property>
+
+<property>
<name>hadoop.security.group.mapping.ldap.connection.timeout.ms</name>
<value>60000</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index 58c2d1a..930c45e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
public class TestGroupsCaching {
- public static final Log LOG = LogFactory.getLog(TestGroupsCaching.class);
+ public static final Log TESTLOG = LogFactory.getLog(TestGroupsCaching.class);
private static String[] myGroups = {"grp1", "grp2"};
private Configuration conf;
@@ -76,7 +76,7 @@ public class TestGroupsCaching {
@Override
public List<String> getGroups(String user) throws IOException {
- LOG.info("Getting groups for " + user);
+ TESTLOG.info("Getting groups for " + user);
delayIfNecessary();
requestCount++;
@@ -115,18 +115,18 @@ public class TestGroupsCaching {
@Override
public void cacheGroupsRefresh() throws IOException {
- LOG.info("Cache is being refreshed.");
+ TESTLOG.info("Cache is being refreshed.");
clearBlackList();
return;
}
public static void clearBlackList() throws IOException {
- LOG.info("Clearing the blacklist");
+ TESTLOG.info("Clearing the blacklist");
blackList.clear();
}
public static void clearAll() throws IOException {
- LOG.info("Resetting FakeGroupMapping");
+ TESTLOG.info("Resetting FakeGroupMapping");
blackList.clear();
allGroups.clear();
requestCount = 0;
@@ -137,12 +137,12 @@ public class TestGroupsCaching {
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
- LOG.info("Adding " + groups + " to groups.");
+ TESTLOG.info("Adding " + groups + " to groups.");
allGroups.addAll(groups);
}
public static void addToBlackList(String user) throws IOException {
- LOG.info("Adding " + user + " to the blacklist");
+ TESTLOG.info("Adding " + user + " to the blacklist");
blackList.add(user);
}
@@ -226,11 +226,12 @@ public class TestGroupsCaching {
// ask for a negative entry
try {
- LOG.error("We are not supposed to get here." + groups.getGroups("user1").toString());
+ TESTLOG.error("We are not supposed to get here."
+ + groups.getGroups("user1").toString());
fail();
} catch (IOException ioe) {
if(!ioe.getMessage().startsWith("No groups found")) {
- LOG.error("Got unexpected exception: " + ioe.getMessage());
+ TESTLOG.error("Got unexpected exception: " + ioe.getMessage());
fail();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8694deb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
index f28cc62..6d9ea08 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedUnixGroupsMapping.java
@@ -22,9 +22,15 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.junit.Test;
+
import static org.junit.Assert.*;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
@@ -32,9 +38,13 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestShellBasedUnixGroupsMapping {
- private static final Log LOG =
+ private static final Log TESTLOG =
LogFactory.getLog(TestShellBasedUnixGroupsMapping.class);
+ private final GenericTestUtils.LogCapturer shellMappingLog =
+ GenericTestUtils.LogCapturer.captureLogs(
+ ShellBasedUnixGroupsMapping.LOG);
+
private class TestGroupUserNotExist
extends ShellBasedUnixGroupsMapping {
/**
@@ -55,7 +65,7 @@ public class TestShellBasedUnixGroupsMapping {
when(executor.getOutput()).thenReturn("");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -90,7 +100,7 @@ public class TestShellBasedUnixGroupsMapping {
when(executor.getOutput()).thenReturn("9999\n9999 abc def");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -133,7 +143,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("23\n23 groupname zzz");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -146,7 +156,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("111\n111 112 113");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -179,7 +189,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("abc\ndef abc hij");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -192,7 +202,7 @@ public class TestShellBasedUnixGroupsMapping {
doNothing().when(executor).execute();
when(executor.getOutput()).thenReturn("1\n1 2 3");
} catch (IOException e) {
- LOG.warn(e.getMessage());
+ TESTLOG.warn(e.getMessage());
}
return executor;
}
@@ -208,6 +218,117 @@ public class TestShellBasedUnixGroupsMapping {
assertTrue(groups.contains("def"));
assertTrue(groups.contains("hij"));
}
+
+ private static class TestDelayedGroupCommand
+ extends ShellBasedUnixGroupsMapping {
+
+ private Long timeoutSecs = 2L;
+
+ TestDelayedGroupCommand() {
+ super();
+ }
+
+ @Override
+ protected String[] getGroupsForUserCommand(String userName) {
+ // Sleeps 2 seconds when executed and writes no output
+ if (Shell.WINDOWS) {
+ return new String[]{"timeout", timeoutSecs.toString()};
+ }
+ return new String[]{"sleep", timeoutSecs.toString()};
+ }
+
+ @Override
+ protected String[] getGroupsIDForUserCommand(String userName) {
+ return getGroupsForUserCommand(userName);
+ }
+ }
+
+ @Test(timeout=4000)
+ public void testFiniteGroupResolutionTime() throws Exception {
+ Configuration conf = new Configuration();
+ String userName = "foobarnonexistinguser";
+ String commandTimeoutMessage =
+ "ran longer than the configured timeout limit";
+ long testTimeout = 1L;
+
+ // Test a 1 second max-runtime timeout
+ conf.setLong(
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS,
+ testTimeout);
+
+ TestDelayedGroupCommand mapping =
+ ReflectionUtils.newInstance(TestDelayedGroupCommand.class, conf);
+
+ ShellCommandExecutor executor = mapping.createGroupExecutor(userName);
+ assertEquals(
+ "Expected the group names executor to carry the configured timeout",
+ testTimeout,
+ executor.getTimeoutInterval());
+
+ executor = mapping.createGroupIDExecutor(userName);
+ assertEquals(
+ "Expected the group ID executor to carry the configured timeout",
+ testTimeout,
+ executor.getTimeoutInterval());
+
+ assertEquals(
+ "Expected no groups to be returned given a shell command timeout",
+ 0,
+ mapping.getGroups(userName).size());
+ assertTrue(
+ "Expected the logs to carry " +
+ "a message about command timeout but was: " +
+ shellMappingLog.getOutput(),
+ shellMappingLog.getOutput().contains(commandTimeoutMessage));
+ shellMappingLog.clearOutput();
+
+ // Test also the parent Groups framework for expected behaviour
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ TestDelayedGroupCommand.class,
+ GroupMappingServiceProvider.class);
+ Groups groups = new Groups(conf);
+ try {
+ groups.getGroups(userName);
+ fail(
+ "The groups framework call should " +
+ "have failed with a command timeout");
+ } catch (IOException e) {
+ assertTrue(
+ "Expected the logs to carry " +
+ "a message about command timeout but was: " +
+ shellMappingLog.getOutput(),
+ shellMappingLog.getOutput().contains(commandTimeoutMessage));
+ }
+ shellMappingLog.clearOutput();
+
+ // Test the no-timeout (default) configuration
+ conf = new Configuration();
+ long defaultTimeout =
+ CommonConfigurationKeys.
+ HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT;
+
+ mapping =
+ ReflectionUtils.newInstance(TestDelayedGroupCommand.class, conf);
+
+ executor = mapping.createGroupExecutor(userName);
+ assertEquals(
+ "Expected the group names executor to carry the default timeout",
+ defaultTimeout,
+ executor.getTimeoutInterval());
+
+ executor = mapping.createGroupIDExecutor(userName);
+ assertEquals(
+ "Expected the group ID executor to carry the default timeout",
+ defaultTimeout,
+ executor.getTimeoutInterval());
+
+ mapping.getGroups(userName);
+ assertFalse(
+ "Didn't expect a timeout of command in execution but logs carry it: " +
+ shellMappingLog.getOutput(),
+ shellMappingLog.getOutput().contains(commandTimeoutMessage));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HADOOP-14091. AbstractFileSystem
implementaion for 'wasbs' scheme. Contributed Varada Hemeswari.
Posted by su...@apache.org.
HADOOP-14091. AbstractFileSystem implementaion for 'wasbs' scheme. Contributed Varada Hemeswari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82607fce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82607fce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82607fce
Branch: refs/heads/YARN-2915
Commit: 82607fce39151fc6ba5bced738088e2bc176dc77
Parents: a4d4a23
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Feb 23 13:48:44 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Feb 23 13:48:44 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/azure/Wasbs.java | 47 ++++++++++++++++
.../fs/azure/TestWasbUriAndConfiguration.java | 57 ++++++++++++++++++++
2 files changed, 104 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82607fce/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java
new file mode 100644
index 0000000..0b4a782
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+/**
+ * WASB implementation of AbstractFileSystem for wasbs scheme.
+ * This impl delegates to the old FileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class Wasbs extends DelegateToFileSystem {
+
+ Wasbs(final URI theUri, final Configuration conf) throws IOException,
+ URISyntaxException {
+ super(theUri, new NativeAzureFileSystem(), conf, "wasbs", false);
+ }
+
+ @Override
+ public int getUriDefaultPort() {
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82607fce/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
index 9d2770e..194a831 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.junit.After;
import org.junit.Assert;
@@ -471,6 +472,62 @@ public class TestWasbUriAndConfiguration {
assertTrue(afs instanceof Wasb);
assertEquals(-1, afs.getUri().getPort());
} finally {
+ testAccount.cleanup();
+ FileSystem.closeAll();
+ }
+ }
+
+ /**
+ * Tests the cases when the scheme specified is 'wasbs'.
+ */
+ @Test
+ public void testAbstractFileSystemImplementationForWasbsScheme() throws Exception {
+ try {
+ testAccount = AzureBlobStorageTestAccount.createMock();
+ Configuration conf = testAccount.getFileSystem().getConf();
+ String authority = testAccount.getFileSystem().getUri().getAuthority();
+ URI defaultUri = new URI("wasbs", authority, null, null, null);
+ conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
+ conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs");
+ conf.addResource("azure-test.xml");
+
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs instanceof NativeAzureFileSystem);
+ assertEquals("wasbs", fs.getScheme());
+
+ AbstractFileSystem afs = FileContext.getFileContext(conf)
+ .getDefaultFileSystem();
+ assertTrue(afs instanceof Wasbs);
+ assertEquals(-1, afs.getUri().getPort());
+ assertEquals("wasbs", afs.getUri().getScheme());
+ } finally {
+ testAccount.cleanup();
+ FileSystem.closeAll();
+ }
+ }
+
+ @Test
+ public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception {
+ try {
+ testAccount = AzureBlobStorageTestAccount.createMock();
+ Configuration conf = testAccount.getFileSystem().getConf();
+ String authority = testAccount.getFileSystem().getUri().getAuthority();
+ URI defaultUri = new URI("wasbs", authority, null, null, null);
+ conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
+
+ FileSystem fs = FileSystem.get(conf);
+ assertTrue(fs instanceof NativeAzureFileSystem);
+ assertEquals("wasbs", fs.getScheme());
+
+ // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified
+ try{
+ FileContext.getFileContext(conf).getDefaultFileSystem();
+ fail("Should've thrown.");
+ }catch(UnsupportedFileSystemException e){
+ }
+
+ } finally {
+ testAccount.cleanup();
FileSystem.closeAll();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-3664. Federation PolicyStore
internal APIs
Posted by su...@apache.org.
YARN-3664. Federation PolicyStore internal APIs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54f9a018
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54f9a018
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54f9a018
Branch: refs/heads/YARN-2915
Commit: 54f9a018dcc725c150fe7167f2c562dc3e8fa6bf
Parents: b2d3d36
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Aug 5 12:34:58 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../federation/store/FederationPolicyStore.java | 76 ++++++++
...SubClusterPoliciesConfigurationsRequest.java | 35 ++++
...ubClusterPoliciesConfigurationsResponse.java | 66 +++++++
...GetSubClusterPolicyConfigurationRequest.java | 62 ++++++
...etSubClusterPolicyConfigurationResponse.java | 65 +++++++
...SetSubClusterPolicyConfigurationRequest.java | 79 ++++++++
...etSubClusterPolicyConfigurationResponse.java | 36 ++++
.../records/SubClusterPolicyConfiguration.java | 130 +++++++++++++
...sterPoliciesConfigurationsRequestPBImpl.java | 95 +++++++++
...terPoliciesConfigurationsResponsePBImpl.java | 191 +++++++++++++++++++
...ClusterPolicyConfigurationRequestPBImpl.java | 103 ++++++++++
...lusterPolicyConfigurationResponsePBImpl.java | 143 ++++++++++++++
.../pb/GetSubClustersInfoResponsePBImpl.java | 4 +-
...ClusterPolicyConfigurationRequestPBImpl.java | 159 +++++++++++++++
...lusterPolicyConfigurationResponsePBImpl.java | 93 +++++++++
.../pb/SubClusterPolicyConfigurationPBImpl.java | 121 ++++++++++++
.../proto/yarn_server_federation_protos.proto | 28 +++
.../records/TestFederationProtocolRecords.java | 53 ++++-
18 files changed, 1536 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
new file mode 100644
index 0000000..9d9bd9b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+
+/**
+ * The FederationPolicyStore provides a key-value interface to access the
+ * policies configured for the system. The key is a "queue" name, i.e., the
+ * system allows to configure a different policy for each queue in the system
+ * (though each policy can make dynamic run-time decisions on a per-job/per-task
+ * basis). The value is a {@code SubClusterPolicyConfiguration}, a serialized
+ * representation of the policy type and its parameters.
+ */
+@Private
+@Unstable
+public interface FederationPolicyStore {
+
+ /**
+ * Get the policy configuration for a given queue.
+ *
+ * @param request the queue whose {@code SubClusterPolicyConfiguration} is
+ * required
+ * @return the {@code SubClusterPolicyConfiguration} for the specified queue
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException;
+
+ /**
+ * Set the policy configuration for a given queue.
+ *
+ * @param request the {@code SubClusterPolicyConfiguration} with the
+ * corresponding queue
+ * @return response empty on successfully updating the
+ * {@code SubClusterPolicyConfiguration} for the specified queue
+ * @throws YarnException if the request is invalid/fails
+ */
+ SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException;
+
+ /**
+ * Get a map of all queue-to-policy configurations.
+ *
+ * @param request empty to represent all configured queues in the system
+ * @return the policies for all currently active queues in the system
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
new file mode 100644
index 0000000..404521b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsRequest.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPoliciesConfigurationsRequest is a request to the
+ * {@code FederationPolicyStore} to obtain all policy configurations.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPoliciesConfigurationsRequest {
+ public GetSubClusterPoliciesConfigurationsRequest newInstance() {
+ return Records.newRecord(GetSubClusterPoliciesConfigurationsRequest.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
new file mode 100644
index 0000000..6554d68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPoliciesConfigurationsResponse.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPolicyConfigurationResponse contains the answer from the {@code
+ * FederationPolicyStore} to a request to get all the policies configured in the
+ * system via a {@link SubClusterPolicyConfiguration}.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPoliciesConfigurationsResponse {
+
+ @Private
+ @Unstable
+ public GetSubClusterPoliciesConfigurationsResponse newInstance(
+ List<SubClusterPolicyConfiguration> policyConfigurations) {
+ GetSubClusterPoliciesConfigurationsResponse response =
+ Records.newRecord(GetSubClusterPoliciesConfigurationsResponse.class);
+ response.setPoliciesConfigs(policyConfigurations);
+ return response;
+ }
+
+ /**
+ * Get all the policies configured in the system.
+ *
+ * @return all the policies configured in the system
+ */
+ @Public
+ @Unstable
+ public abstract List<SubClusterPolicyConfiguration> getPoliciesConfigs();
+
+ /**
+ * Sets all the policies configured in the system.
+ *
+ * @param policyConfigurations all the policies configured in the system
+ */
+ @Private
+ @Unstable
+ public abstract void setPoliciesConfigs(
+ List<SubClusterPolicyConfiguration> policyConfigurations);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
new file mode 100644
index 0000000..7b7d8c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationRequest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPolicyConfigurationRequest is a request to the
+ * {@code FederationPolicyStore} to get the configuration of a policy for a
+ * given queue.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPolicyConfigurationRequest {
+
+ @Private
+ @Unstable
+ public GetSubClusterPolicyConfigurationRequest newInstance(String queueName) {
+ GetSubClusterPolicyConfigurationRequest request =
+ Records.newRecord(GetSubClusterPolicyConfigurationRequest.class);
+ request.setQueue(queueName);
+ return request;
+ }
+
+ /**
+ * Get the name of the queue for which we are requesting a policy
+ * configuration.
+ *
+ * @return the name of the queue
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Sets the name of the queue for which we are requesting a policy
+ * configuration.
+ *
+ * @param queueName the name of the queue
+ */
+ @Private
+ @Unstable
+ public abstract void setQueue(String queueName);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
new file mode 100644
index 0000000..11a46e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClusterPolicyConfigurationResponse.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * GetSubClusterPolicyConfigurationResponse contains the answer from the {@code
+ * FederationPolicyStore} to a request to get the information about how a policy
+ * should be configured via a {@link SubClusterPolicyConfiguration}.
+ */
+@Private
+@Unstable
+public abstract class GetSubClusterPolicyConfigurationResponse {
+
+ @Private
+ @Unstable
+ public GetSubClusterPolicyConfigurationResponse newInstance(
+ SubClusterPolicyConfiguration policy) {
+ GetSubClusterPolicyConfigurationResponse response =
+ Records.newRecord(GetSubClusterPolicyConfigurationResponse.class);
+ response.setPolicyConfiguration(policy);
+ return response;
+ }
+
+ /**
+ * Get the policy configuration.
+ *
+ * @return the policy configuration for the specified queue
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterPolicyConfiguration getPolicyConfiguration();
+
+ /**
+ * Sets the policyConfiguration configuration.
+ *
+ * @param policyConfiguration the policyConfiguration configuration for the
+ * specified queue
+ */
+ @Private
+ @Unstable
+ public abstract void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
new file mode 100644
index 0000000..06d5399
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationRequest.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SetSubClusterPolicyConfigurationRequest is a request to the
+ * {@code FederationPolicyStore} to set the policy configuration corresponding
+ * to a queue.
+ */
+@Private
+@Unstable
+public abstract class SetSubClusterPolicyConfigurationRequest {
+ @Private
+ @Unstable
+ public SetSubClusterPolicyConfigurationRequest newInstance(
+ SubClusterPolicyConfiguration policy) {
+ SetSubClusterPolicyConfigurationRequest request =
+ Records.newRecord(SetSubClusterPolicyConfigurationRequest.class);
+ request.setPolicyConfiguration(policy);
+ return request;
+ }
+
+ /**
+ * Get the name of the queue for which we are configuring a policy.
+ *
+ * @return the name of the queue
+ */
+ @Public
+ @Unstable
+ public abstract String getQueue();
+
+ /**
+ * Sets the name of the queue for which we are configuring a policy.
+ *
+ * @param queueName the name of the queue
+ */
+ @Private
+ @Unstable
+ public abstract void setQueue(String queueName);
+
+ /**
+ * Get the policy configuration assigned to the queue.
+ *
+ * @return the policy for the specified queue
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterPolicyConfiguration getPolicyConfiguration();
+
+ /**
+ * Set the policyConfiguration configuration for the queue.
+ *
+ * @param policyConfiguration the policyConfiguration for the specified queue
+ */
+ @Private
+ @Unstable
+ public abstract void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
new file mode 100644
index 0000000..33c4043
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SetSubClusterPolicyConfigurationResponse.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SetSubClusterPolicyConfigurationResponse contains the answer from the
+ * {@code FederationPolicyStore} to a request to set for a policy configuration
+ * for a given queue.
+ */
+@Private
+@Unstable
+public abstract class SetSubClusterPolicyConfigurationResponse {
+ public SetSubClusterPolicyConfigurationResponse newInstance() {
+ return Records.newRecord(SetSubClusterPolicyConfigurationResponse.class);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
new file mode 100644
index 0000000..bc12acb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.nio.ByteBuffer;
+
+// used in javadoc
+
+/**
+ * {@link SubClusterPolicyConfiguration} is a class that represents a
+ * configuration of a policy. It contains a policy type (resolve to a class
+ * name) and its params as an opaque {@link ByteBuffer}.
+ *
+ * Note: by design the params are an opaque ByteBuffer, this allows for enough
+ * flexibility to evolve the policies without impacting the protocols to/from
+ * the federation state store.
+ */
+@Private
+@Unstable
+public abstract class SubClusterPolicyConfiguration {
+
+ @Private
+ @Unstable
+ public static SubClusterPolicyConfiguration newInstance(String policyType,
+ ByteBuffer policyParams) {
+ SubClusterPolicyConfiguration policy =
+ Records.newRecord(SubClusterPolicyConfiguration.class);
+ policy.setType(policyType);
+ policy.setParams(policyParams);
+ return policy;
+ }
+
+ /**
+ * Get the type of the policy. This could be random, round-robin, load-based,
+ * etc.
+ *
+ * @return the type of the policy
+ */
+ @Public
+ @Unstable
+ public abstract String getType();
+
+ /**
+ * Sets the type of the policy. This could be random, round-robin, load-based,
+ * etc.
+ *
+ * @param policyType the type of the policy
+ */
+ @Private
+ @Unstable
+ public abstract void setType(String policyType);
+
+ /**
+ * Get the policy parameters. This affects how the policy behaves and an
+ * example could be weight distribution of queues across multiple
+ * sub-clusters.
+ *
+ * @return the byte array that contains the parameters
+ */
+ @Public
+ @Unstable
+ public abstract ByteBuffer getParams();
+
+ /**
+ * Set the policy parameters. This affects how the policy behaves and an
+ * example could be weight distribution of queues across multiple
+ * sub-clusters.
+ *
+ * @param policyParams byte array that describes the policy
+ */
+ @Private
+ @Unstable
+ public abstract void setParams(ByteBuffer policyParams);
+
+ @Override
+ public int hashCode() {
+ return 31 * getParams().hashCode() + getType().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ SubClusterPolicyConfiguration other = (SubClusterPolicyConfiguration) obj;
+ if (!this.getType().equals(other.getType())) {
+ return false;
+ }
+ if (!this.getParams().equals(other.getParams())) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getType());
+ sb.append(" : ");
+ sb.append(getParams());
+ return sb.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java
new file mode 100644
index 0000000..3cb6601
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsRequestPBImpl.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsRequestProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPoliciesConfigurationsRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPoliciesConfigurationsRequestPBImpl
+ extends GetSubClusterPoliciesConfigurationsRequest {
+
+ private GetSubClusterPoliciesConfigurationsRequestProto proto =
+ GetSubClusterPoliciesConfigurationsRequestProto.getDefaultInstance();
+ private GetSubClusterPoliciesConfigurationsRequestProto.Builder builder =
+ null;
+ private boolean viaProto = false;
+
+ public GetSubClusterPoliciesConfigurationsRequestPBImpl() {
+ builder = GetSubClusterPoliciesConfigurationsRequestProto.newBuilder();
+ }
+
+ public GetSubClusterPoliciesConfigurationsRequestPBImpl(
+ GetSubClusterPoliciesConfigurationsRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPoliciesConfigurationsRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder =
+ GetSubClusterPoliciesConfigurationsRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
new file mode 100644
index 0000000..67c3654
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPoliciesConfigurationsResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPoliciesConfigurationsResponsePBImpl
+ extends GetSubClusterPoliciesConfigurationsResponse {
+
+ private GetSubClusterPoliciesConfigurationsResponseProto proto =
+ GetSubClusterPoliciesConfigurationsResponseProto.getDefaultInstance();
+ private GetSubClusterPoliciesConfigurationsResponseProto.Builder builder =
+ null;
+ private boolean viaProto = false;
+
+ private List<SubClusterPolicyConfiguration> subClusterPolicies = null;
+
+ public GetSubClusterPoliciesConfigurationsResponsePBImpl() {
+ builder = GetSubClusterPoliciesConfigurationsResponseProto.newBuilder();
+ }
+
+ public GetSubClusterPoliciesConfigurationsResponsePBImpl(
+ GetSubClusterPoliciesConfigurationsResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPoliciesConfigurationsResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder =
+ GetSubClusterPoliciesConfigurationsResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterPolicies != null) {
+ addSubClusterPoliciesConfigurationsToProto();
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public List<SubClusterPolicyConfiguration> getPoliciesConfigs() {
+ initSubClusterPoliciesConfigurationsList();
+ return this.subClusterPolicies;
+ }
+
+ @Override
+ public void setPoliciesConfigs(
+ List<SubClusterPolicyConfiguration> policyConfigurations) {
+ maybeInitBuilder();
+ if (policyConfigurations == null) {
+ builder.clearPoliciesConfigurations();
+ }
+ this.subClusterPolicies = policyConfigurations;
+ }
+
+ private void initSubClusterPoliciesConfigurationsList() {
+ if (this.subClusterPolicies != null) {
+ return;
+ }
+ GetSubClusterPoliciesConfigurationsResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<SubClusterPolicyConfigurationProto> subClusterPoliciesList =
+ p.getPoliciesConfigurationsList();
+ subClusterPolicies = new ArrayList<SubClusterPolicyConfiguration>();
+
+ for (SubClusterPolicyConfigurationProto r : subClusterPoliciesList) {
+ subClusterPolicies.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addSubClusterPoliciesConfigurationsToProto() {
+ maybeInitBuilder();
+ builder.clearPoliciesConfigurations();
+ if (subClusterPolicies == null) {
+ return;
+ }
+ Iterable<SubClusterPolicyConfigurationProto> iterable =
+ new Iterable<SubClusterPolicyConfigurationProto>() {
+ @Override
+ public Iterator<SubClusterPolicyConfigurationProto> iterator() {
+ return new Iterator<SubClusterPolicyConfigurationProto>() {
+
+ private Iterator<SubClusterPolicyConfiguration> iter =
+ subClusterPolicies.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public SubClusterPolicyConfigurationProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllPoliciesConfigurations(iterable);
+ }
+
+ private SubClusterPolicyConfiguration convertFromProtoFormat(
+ SubClusterPolicyConfigurationProto policy) {
+ return new SubClusterPolicyConfigurationPBImpl(policy);
+ }
+
+ private SubClusterPolicyConfigurationProto convertToProtoFormat(
+ SubClusterPolicyConfiguration policy) {
+ return ((SubClusterPolicyConfigurationPBImpl) policy).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java
new file mode 100644
index 0000000..35aff17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPolicyConfigurationRequest}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPolicyConfigurationRequestPBImpl
+ extends GetSubClusterPolicyConfigurationRequest {
+
+ private GetSubClusterPolicyConfigurationRequestProto proto =
+ GetSubClusterPolicyConfigurationRequestProto.getDefaultInstance();
+ private GetSubClusterPolicyConfigurationRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetSubClusterPolicyConfigurationRequestPBImpl() {
+ builder = GetSubClusterPolicyConfigurationRequestProto.newBuilder();
+ }
+
+ public GetSubClusterPolicyConfigurationRequestPBImpl(
+ GetSubClusterPolicyConfigurationRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPolicyConfigurationRequestProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterPolicyConfigurationRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public String getQueue() {
+ GetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearQueue();
+ return;
+ }
+ builder.setQueue(queueName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
new file mode 100644
index 0000000..96b88ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetSubClusterPolicyConfigurationResponse}.
+ */
+@Private
+@Unstable
+public class GetSubClusterPolicyConfigurationResponsePBImpl
+ extends GetSubClusterPolicyConfigurationResponse {
+
+ private GetSubClusterPolicyConfigurationResponseProto proto =
+ GetSubClusterPolicyConfigurationResponseProto.getDefaultInstance();
+ private GetSubClusterPolicyConfigurationResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterPolicyConfiguration subClusterPolicy = null;
+
+ public GetSubClusterPolicyConfigurationResponsePBImpl() {
+ builder = GetSubClusterPolicyConfigurationResponseProto.newBuilder();
+ }
+
+ public GetSubClusterPolicyConfigurationResponsePBImpl(
+ GetSubClusterPolicyConfigurationResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetSubClusterPolicyConfigurationResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetSubClusterPolicyConfigurationResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterPolicy != null
+ && !((SubClusterPolicyConfigurationPBImpl) this.subClusterPolicy)
+ .getProto().equals(builder.getPolicyConfiguration())) {
+ builder
+ .setPolicyConfiguration(convertToProtoFormat(this.subClusterPolicy));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration getPolicyConfiguration() {
+ GetSubClusterPolicyConfigurationResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.subClusterPolicy != null) {
+ return this.subClusterPolicy;
+ }
+ if (!p.hasPolicyConfiguration()) {
+ return null;
+ }
+ this.subClusterPolicy = convertFromProtoFormat(p.getPolicyConfiguration());
+ return this.subClusterPolicy;
+ }
+
+ @Override
+ public void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration) {
+ maybeInitBuilder();
+ if (policyConfiguration == null) {
+ builder.clearPolicyConfiguration();
+ }
+ this.subClusterPolicy = policyConfiguration;
+ }
+
+ private SubClusterPolicyConfiguration convertFromProtoFormat(
+ SubClusterPolicyConfigurationProto policy) {
+ return new SubClusterPolicyConfigurationPBImpl(policy);
+ }
+
+ private SubClusterPolicyConfigurationProto convertToProtoFormat(
+ SubClusterPolicyConfiguration policy) {
+ return ((SubClusterPolicyConfigurationPBImpl) policy).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
index 92bdf06..2efa3b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -66,7 +66,7 @@ public class GetSubClustersInfoResponsePBImpl
private void mergeLocalToBuilder() {
if (this.subClusterInfos != null) {
- addReservationResourcesToProto();
+ addSubClusterInfosToProto();
}
}
@@ -114,7 +114,7 @@ public class GetSubClustersInfoResponsePBImpl
}
}
- private void addReservationResourcesToProto() {
+ private void addSubClusterInfosToProto() {
maybeInitBuilder();
builder.clearSubClusterInfos();
if (subClusterInfos == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
new file mode 100644
index 0000000..5e29bd5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link SetSubClusterPolicyConfigurationRequest}.
+ */
+@Private
+@Unstable
+public class SetSubClusterPolicyConfigurationRequestPBImpl
+ extends SetSubClusterPolicyConfigurationRequest {
+
+ private SetSubClusterPolicyConfigurationRequestProto proto =
+ SetSubClusterPolicyConfigurationRequestProto.getDefaultInstance();
+ private SetSubClusterPolicyConfigurationRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private SubClusterPolicyConfiguration subClusterPolicy = null;
+
+ public SetSubClusterPolicyConfigurationRequestPBImpl() {
+ builder = SetSubClusterPolicyConfigurationRequestProto.newBuilder();
+ }
+
+ public SetSubClusterPolicyConfigurationRequestPBImpl(
+ SetSubClusterPolicyConfigurationRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SetSubClusterPolicyConfigurationRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SetSubClusterPolicyConfigurationRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.subClusterPolicy != null
+ && !((SubClusterPolicyConfigurationPBImpl) this.subClusterPolicy)
+ .getProto().equals(builder.getPolicyConfiguration())) {
+ builder
+ .setPolicyConfiguration(convertToProtoFormat(this.subClusterPolicy));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public String getQueue() {
+ SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queueName) {
+ maybeInitBuilder();
+ if (queueName == null) {
+ builder.clearQueue();
+ return;
+ }
+ builder.setQueue(queueName);
+ }
+
+ @Override
+ public SubClusterPolicyConfiguration getPolicyConfiguration() {
+ SetSubClusterPolicyConfigurationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.subClusterPolicy != null) {
+ return this.subClusterPolicy;
+ }
+ if (!p.hasPolicyConfiguration()) {
+ return null;
+ }
+ this.subClusterPolicy = convertFromProtoFormat(p.getPolicyConfiguration());
+ return this.subClusterPolicy;
+ }
+
+ @Override
+ public void setPolicyConfiguration(
+ SubClusterPolicyConfiguration policyConfiguration) {
+ maybeInitBuilder();
+ if (policyConfiguration == null) {
+ builder.clearPolicyConfiguration();
+ }
+ this.subClusterPolicy = policyConfiguration;
+ }
+
+ private SubClusterPolicyConfiguration convertFromProtoFormat(
+ SubClusterPolicyConfigurationProto policy) {
+ return new SubClusterPolicyConfigurationPBImpl(policy);
+ }
+
+ private SubClusterPolicyConfigurationProto convertToProtoFormat(
+ SubClusterPolicyConfiguration policy) {
+ return ((SubClusterPolicyConfigurationPBImpl) policy).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java
new file mode 100644
index 0000000..9d79215
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationResponsePBImpl.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link SetSubClusterPolicyConfigurationResponse}.
+ */
+@Private
+@Unstable
+public class SetSubClusterPolicyConfigurationResponsePBImpl
+ extends SetSubClusterPolicyConfigurationResponse {
+
+ private SetSubClusterPolicyConfigurationResponseProto proto =
+ SetSubClusterPolicyConfigurationResponseProto.getDefaultInstance();
+ private SetSubClusterPolicyConfigurationResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SetSubClusterPolicyConfigurationResponsePBImpl() {
+ builder = SetSubClusterPolicyConfigurationResponseProto.newBuilder();
+ }
+
+ public SetSubClusterPolicyConfigurationResponsePBImpl(
+ SetSubClusterPolicyConfigurationResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SetSubClusterPolicyConfigurationResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SetSubClusterPolicyConfigurationResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
new file mode 100644
index 0000000..fe9d9db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterPolicyConfigurationPBImpl.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protobuf based implementation of {@link SubClusterPolicyConfiguration}.
+ *
+ */
+@Private
+@Unstable
+public class SubClusterPolicyConfigurationPBImpl
+ extends SubClusterPolicyConfiguration {
+
+ private SubClusterPolicyConfigurationProto proto =
+ SubClusterPolicyConfigurationProto.getDefaultInstance();
+ private SubClusterPolicyConfigurationProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterPolicyConfigurationPBImpl() {
+ builder = SubClusterPolicyConfigurationProto.newBuilder();
+ }
+
+ public SubClusterPolicyConfigurationPBImpl(
+ SubClusterPolicyConfigurationProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SubClusterPolicyConfigurationProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SubClusterPolicyConfigurationProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public String getType() {
+ SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getType();
+ }
+
+ @Override
+ public void setType(String policyType) {
+ maybeInitBuilder();
+ if (policyType == null) {
+ builder.clearType();
+ return;
+ }
+ builder.setType(policyType);
+ }
+
+ @Override
+ public ByteBuffer getParams() {
+ SubClusterPolicyConfigurationProtoOrBuilder p = viaProto ? proto : builder;
+ return ProtoUtils.convertFromProtoFormat(p.getParams());
+ }
+
+ @Override
+ public void setParams(ByteBuffer policyParams) {
+ maybeInitBuilder();
+ if (policyParams == null) {
+ builder.clearParams();
+ return;
+ }
+ builder.setParams(ProtoUtils.convertToProtoFormat(policyParams));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index b1ad310..3f1cee9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -134,3 +134,31 @@ message DeleteApplicationHomeSubClusterRequestProto {
message DeleteApplicationHomeSubClusterResponseProto {
}
+
+message SubClusterPolicyConfigurationProto {
+ optional string type = 1;
+ optional bytes params = 2;
+}
+
+message GetSubClusterPolicyConfigurationRequestProto {
+ optional string queue = 1;
+}
+
+message GetSubClusterPolicyConfigurationResponseProto {
+ optional SubClusterPolicyConfigurationProto policy_configuration = 1;
+}
+
+message SetSubClusterPolicyConfigurationRequestProto {
+ optional string queue = 1;
+ optional SubClusterPolicyConfigurationProto policy_configuration = 2;
+}
+
+message SetSubClusterPolicyConfigurationResponseProto {
+}
+
+message GetSubClusterPoliciesConfigurationsRequestProto {
+}
+
+message GetSubClusterPoliciesConfigurationsResponseProto {
+ repeated SubClusterPolicyConfigurationProto policies_configurations = 1;
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54f9a018/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
index 210a246..cf8cf71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -29,8 +29,14 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApp
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPoliciesConfigurationsResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterPolicyConfigurationResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SetSubClusterPolicyConfigurationResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterHeartbeatRequestProto;
@@ -51,8 +57,14 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplica
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationsHomeSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPoliciesConfigurationsRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPoliciesConfigurationsResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPolicyConfigurationRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterPolicyConfigurationResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SetSubClusterPolicyConfigurationRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SetSubClusterPolicyConfigurationResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterDeregisterResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterHeartbeatRequestPBImpl;
@@ -79,6 +91,7 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
generateByNewInstance(SubClusterId.class);
generateByNewInstance(SubClusterInfo.class);
generateByNewInstance(ApplicationHomeSubCluster.class);
+ generateByNewInstance(SubClusterPolicyConfiguration.class);
}
@Test
@@ -211,4 +224,42 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
DeleteApplicationHomeSubClusterResponseProto.class);
}
-}
\ No newline at end of file
+ @Test
+ public void testGetSubClusterPolicyConfigurationRequest() throws Exception {
+ validatePBImplRecord(GetSubClusterPolicyConfigurationRequestPBImpl.class,
+ GetSubClusterPolicyConfigurationRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterPolicyConfigurationResponse() throws Exception {
+ validatePBImplRecord(GetSubClusterPolicyConfigurationResponsePBImpl.class,
+ GetSubClusterPolicyConfigurationResponseProto.class);
+ }
+
+ @Test
+ public void testSetSubClusterPolicyConfigurationRequest() throws Exception {
+ validatePBImplRecord(SetSubClusterPolicyConfigurationRequestPBImpl.class,
+ SetSubClusterPolicyConfigurationRequestProto.class);
+ }
+
+ @Test
+ public void testSetSubClusterPolicyConfigurationResponse() throws Exception {
+ validatePBImplRecord(SetSubClusterPolicyConfigurationResponsePBImpl.class,
+ SetSubClusterPolicyConfigurationResponseProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterPoliciesConfigurationsRequest()
+ throws Exception {
+ validatePBImplRecord(GetSubClusterPoliciesConfigurationsRequestPBImpl.class,
+ GetSubClusterPoliciesConfigurationsRequestProto.class);
+ }
+
+ @Test
+ public void testGetSubClusterPoliciesConfigurationsResponse()
+ throws Exception {
+ validatePBImplRecord(
+ GetSubClusterPoliciesConfigurationsResponsePBImpl.class,
+ GetSubClusterPoliciesConfigurationsResponseProto.class);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-5390. Federation Subcluster
Resolver. Contributed by Ellen Hui.
Posted by su...@apache.org.
YARN-5390. Federation Subcluster Resolver. Contributed by Ellen Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/934c3d50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/934c3d50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/934c3d50
Branch: refs/heads/YARN-2915
Commit: 934c3d50edd28100039851c115cde0c7e2cbafbb
Parents: 5861f61
Author: Subru Krishnan <su...@apache.org>
Authored: Thu Aug 4 15:58:31 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../src/main/resources/yarn-default.xml | 7 +
.../hadoop-yarn-server-common/pom.xml | 10 +
.../resolver/AbstractSubClusterResolver.java | 67 +++++++
.../resolver/DefaultSubClusterResolverImpl.java | 164 +++++++++++++++++
.../federation/resolver/SubClusterResolver.java | 58 ++++++
.../federation/resolver/package-info.java | 17 ++
.../resolver/TestDefaultSubClusterResolver.java | 184 +++++++++++++++++++
.../src/test/resources/nodes | 4 +
.../src/test/resources/nodes-malformed | 3 +
10 files changed, 522 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cdccec6..8901797 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2485,6 +2485,14 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20;
////////////////////////////////
+ // Federation Configs
+ ////////////////////////////////
+
+ public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+ public static final String FEDERATION_MACHINE_LIST =
+ FEDERATION_PREFIX + "machine-list";
+
+ ////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 368946e..09c4309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2616,6 +2616,13 @@
<!-- Other Configuration -->
<property>
+ <description>
+ Machine list file to be loaded by the FederationSubCluster Resolver
+ </description>
+ <name>yarn.federation.machine-list</name>
+ </property>
+
+ <property>
<description>The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 9cc3cae..6d2fbef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -177,6 +177,16 @@
</filesets>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/nodes</exclude>
+ <exclude>src/test/resources/nodes-malformed</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
new file mode 100644
index 0000000..8238633
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
@@ -0,0 +1,67 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import java.util.HashMap;
+import java.util.Set;
+import java.util.Map;
+
+/**
+ * Partial implementation of {@link SubClusterResolver}, containing basic
+ * implementations of the read methods.
+ */
+public abstract class AbstractSubClusterResolver implements SubClusterResolver {
+ private Map<String, SubClusterId> nodeToSubCluster =
+ new HashMap<String, SubClusterId>();
+ private Map<String, Set<SubClusterId>> rackToSubClusters =
+ new HashMap<String, Set<SubClusterId>>();
+
+ @Override
+ public SubClusterId getSubClusterForNode(String nodename)
+ throws YarnException {
+ SubClusterId subClusterId = this.nodeToSubCluster.get(nodename);
+
+ if (subClusterId == null) {
+ throw new YarnException("Cannot find subClusterId for node " + nodename);
+ }
+
+ return subClusterId;
+ }
+
+ @Override
+ public Set<SubClusterId> getSubClustersForRack(String rackname)
+ throws YarnException {
+ if (!rackToSubClusters.containsKey(rackname)) {
+ throw new YarnException("Cannot resolve rack " + rackname);
+ }
+
+ return rackToSubClusters.get(rackname);
+ }
+
+ protected Map<String, SubClusterId> getNodeToSubCluster() {
+ return nodeToSubCluster;
+ }
+
+ protected Map<String, Set<SubClusterId>> getRackToSubClusters() {
+ return rackToSubClusters;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java
new file mode 100644
index 0000000..d3c5c26
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/DefaultSubClusterResolverImpl.java
@@ -0,0 +1,164 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import java.io.BufferedReader;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Default simple sub-cluster and rack resolver class.
+ *
+ * This class expects a three-column comma separated file, specified in
+ * yarn.federation.machine-list. Each line of the file should be of the format:
+ *
+ * nodeName, subClusterId, rackName
+ *
+ * Lines that do not follow this format will be ignored. This resolver only
+ * loads the file when load() is explicitly called; it will not react to changes
+ * to the file.
+ *
+ * It is case-insensitive on the rack and node names and ignores
+ * leading/trailing whitespace.
+ *
+ */
+public class DefaultSubClusterResolverImpl extends AbstractSubClusterResolver
+ implements SubClusterResolver {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultSubClusterResolverImpl.class);
+ private Configuration conf;
+
+ // Index of the node hostname in the machine info file.
+ private static final int NODE_NAME_INDEX = 0;
+
+ // Index of the sub-cluster ID in the machine info file.
+ private static final int SUBCLUSTER_ID_INDEX = 1;
+
+ // Index of the rack name ID in the machine info file.
+ private static final int RACK_NAME_INDEX = 2;
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public SubClusterId getSubClusterForNode(String nodename)
+ throws YarnException {
+ return super.getSubClusterForNode(nodename.toUpperCase());
+ }
+
+ @Override
+ public void load() {
+ String fileName =
+ this.conf.get(YarnConfiguration.FEDERATION_MACHINE_LIST, "");
+
+ try {
+ if (fileName == null || fileName.trim().length() == 0) {
+ LOG.info(
+ "The machine list file path is not specified in the configuration");
+ return;
+ }
+
+ Path file = null;
+ BufferedReader reader = null;
+
+ try {
+ file = Paths.get(fileName);
+ } catch (InvalidPathException e) {
+ LOG.info("The configured machine list file path {} does not exist",
+ fileName);
+ return;
+ }
+
+ try {
+ reader = Files.newBufferedReader(file, Charset.defaultCharset());
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ String[] tokens = line.split(",");
+ if (tokens.length == 3) {
+
+ String nodeName = tokens[NODE_NAME_INDEX].trim().toUpperCase();
+ SubClusterId subClusterId =
+ SubClusterId.newInstance(tokens[SUBCLUSTER_ID_INDEX].trim());
+ String rackName = tokens[RACK_NAME_INDEX].trim().toUpperCase();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading node into resolver: {} --> {}", nodeName,
+ subClusterId);
+ LOG.debug("Loading rack into resolver: {} --> {} ", rackName,
+ subClusterId);
+ }
+
+ this.getNodeToSubCluster().put(nodeName, subClusterId);
+ loadRackToSubCluster(rackName, subClusterId);
+ } else {
+ LOG.warn("Skipping malformed line in machine list: " + line);
+ }
+ }
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ LOG.info("Successfully loaded file {}", fileName);
+
+ } catch (Exception e) {
+ LOG.error("Failed to parse file " + fileName, e);
+ }
+ }
+
+ private void loadRackToSubCluster(String rackName,
+ SubClusterId subClusterId) {
+ String rackNameUpper = rackName.toUpperCase();
+
+ if (!this.getRackToSubClusters().containsKey(rackNameUpper)) {
+ this.getRackToSubClusters().put(rackNameUpper,
+ new HashSet<SubClusterId>());
+ }
+
+ this.getRackToSubClusters().get(rackNameUpper).add(subClusterId);
+
+ }
+
+ @Override
+ public Set<SubClusterId> getSubClustersForRack(String rackname)
+ throws YarnException {
+ return super.getSubClustersForRack(rackname.toUpperCase());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java
new file mode 100644
index 0000000..c6adfa6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/SubClusterResolver.java
@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * An utility that helps to determine the sub-cluster that a specified node
+ * belongs to.
+ */
+public interface SubClusterResolver extends Configurable {
+
+ /**
+ * Obtain the sub-cluster that a specified node belongs to.
+ *
+ * @param nodename the node whose sub-cluster is to be determined
+ * @return the sub-cluster as identified by the {@link SubClusterId} that the
+ * node belongs to
+ * @throws YarnException if the node's sub-cluster cannot be resolved
+ */
+ SubClusterId getSubClusterForNode(String nodename) throws YarnException;
+
+ /**
+ * Obtain the sub-clusters that have nodes on a specified rack.
+ *
+ * @param rackname the name of the rack
+ * @return the sub-clusters as identified by the {@link SubClusterId} that
+ * have nodes on the given rack
+ * @throws YarnException if the sub-cluster of any node on the rack cannot be
+ * resolved, or if the rack name is not recognized
+ */
+ Set<SubClusterId> getSubClustersForRack(String rackname) throws YarnException;
+
+ /**
+ * Load the nodes to subCluster mapping from the file.
+ */
+ void load();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java
new file mode 100644
index 0000000..c042660
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.resolver;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java
new file mode 100644
index 0000000..7396942
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/resolver/TestDefaultSubClusterResolver.java
@@ -0,0 +1,184 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.federation.resolver;
+
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test {@link SubClusterResolver} against correct and malformed Federation
+ * machine lists.
+ */
+public class TestDefaultSubClusterResolver {
+ private static YarnConfiguration conf;
+ private static SubClusterResolver resolver;
+
+ public static void setUpGoodFile() {
+ conf = new YarnConfiguration();
+ resolver = new DefaultSubClusterResolverImpl();
+
+ URL url =
+ Thread.currentThread().getContextClassLoader().getResource("nodes");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'nodes' dummy file in classpath");
+ }
+
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, url.getPath());
+ resolver.setConf(conf);
+ resolver.load();
+ }
+
+ private void setUpMalformedFile() {
+ conf = new YarnConfiguration();
+ resolver = new DefaultSubClusterResolverImpl();
+
+ URL url = Thread.currentThread().getContextClassLoader()
+ .getResource("nodes-malformed");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find 'nodes-malformed' dummy file in classpath");
+ }
+
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, url.getPath());
+ resolver.setConf(conf);
+ resolver.load();
+ }
+
+ private void setUpNonExistentFile() {
+ conf = new YarnConfiguration();
+ resolver = new DefaultSubClusterResolverImpl();
+
+ conf.set(YarnConfiguration.FEDERATION_MACHINE_LIST, "fileDoesNotExist");
+ resolver.setConf(conf);
+ resolver.load();
+ }
+
+ @Test
+ public void testGetSubClusterForNode() throws YarnException {
+ setUpGoodFile();
+
+ // All lowercase, no whitespace in machine list file
+ Assert.assertEquals(SubClusterId.newInstance("subcluster1"),
+ resolver.getSubClusterForNode("node1"));
+ // Leading and trailing whitespace in machine list file
+ Assert.assertEquals(SubClusterId.newInstance("subcluster2"),
+ resolver.getSubClusterForNode("node2"));
+ // Node name capitalization in machine list file
+ Assert.assertEquals(SubClusterId.newInstance("subcluster3"),
+ resolver.getSubClusterForNode("node3"));
+
+ try {
+ resolver.getSubClusterForNode("nodeDoesNotExist");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterForNodeMalformedFile() throws YarnException {
+ setUpMalformedFile();
+
+ try {
+ resolver.getSubClusterForNode("node1");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+
+ try {
+ resolver.getSubClusterForNode("node2");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+
+ Assert.assertEquals(SubClusterId.newInstance("subcluster3"),
+ resolver.getSubClusterForNode("node3"));
+
+ try {
+ resolver.getSubClusterForNode("nodeDoesNotExist");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+ }
+
+ @Test
+ public void testGetSubClusterForNodeNoFile() throws YarnException {
+ setUpNonExistentFile();
+
+ try {
+ resolver.getSubClusterForNode("node1");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().startsWith("Cannot find subClusterId for node"));
+ }
+ }
+
+ @Test
+ public void testGetSubClustersForRack() throws YarnException {
+ setUpGoodFile();
+
+ Set<SubClusterId> rack1Expected = new HashSet<SubClusterId>();
+ rack1Expected.add(SubClusterId.newInstance("subcluster1"));
+ rack1Expected.add(SubClusterId.newInstance("subcluster2"));
+
+ Set<SubClusterId> rack2Expected = new HashSet<SubClusterId>();
+ rack2Expected.add(SubClusterId.newInstance("subcluster3"));
+
+ // Two subclusters have nodes in rack1
+ Assert.assertEquals(rack1Expected, resolver.getSubClustersForRack("rack1"));
+
+ // Two nodes are in rack2, but both belong to subcluster3
+ Assert.assertEquals(rack2Expected, resolver.getSubClustersForRack("rack2"));
+
+ try {
+ resolver.getSubClustersForRack("rackDoesNotExist");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Cannot resolve rack"));
+ }
+ }
+
+ @Test
+ public void testGetSubClustersForRackNoFile() throws YarnException {
+ setUpNonExistentFile();
+
+ try {
+ resolver.getSubClustersForRack("rack1");
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(e.getMessage().startsWith("Cannot resolve rack"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
new file mode 100644
index 0000000..e4d6112
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes
@@ -0,0 +1,4 @@
+node1,subcluster1,rack1
+ node2 , subcluster2, RACK1
+noDE3,subcluster3, rack2
+node4, subcluster3, rack2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/934c3d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed
new file mode 100644
index 0000000..6d0aa39
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/resources/nodes-malformed
@@ -0,0 +1,3 @@
+node1,
+node2,subcluster2,subCluster2, rack1
+node3,subcluster3, rack2
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-4779. Fix AM container
allocation logic in SLS. Contributed by Wangda Tan.
Posted by su...@apache.org.
YARN-4779. Fix AM container allocation logic in SLS. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b32ffa27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b32ffa27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b32ffa27
Branch: refs/heads/YARN-2915
Commit: b32ffa2753e83615b980721b6067fcc35ce54372
Parents: e8694de
Author: Sunil G <su...@apache.org>
Authored: Fri Feb 24 21:39:25 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Fri Feb 24 21:39:25 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 20 +-
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 89 +++++---
.../yarn/sls/appmaster/MRAMSimulator.java | 218 ++++++++-----------
.../sls/resourcemanager/MockAMLauncher.java | 115 ++++++++++
.../sls/scheduler/SLSCapacityScheduler.java | 24 ++
5 files changed, 305 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 61738fb..61b7f36 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -55,12 +56,14 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
+import org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper;
import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
@@ -119,10 +122,10 @@ public class SLSRunner {
this.printSimulation = printsimulation;
metricsOutputDir = outputDir;
- nmMap = new HashMap<NodeId, NMSimulator>();
- queueAppNumMap = new HashMap<String, Integer>();
- amMap = new HashMap<String, AMSimulator>();
- amClassMap = new HashMap<String, Class>();
+ nmMap = new HashMap<>();
+ queueAppNumMap = new HashMap<>();
+ amMap = new ConcurrentHashMap<>();
+ amClassMap = new HashMap<>();
// runner configuration
conf = new Configuration(false);
@@ -179,7 +182,14 @@ public class SLSRunner {
}
rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
- rm = new ResourceManager();
+
+ final SLSRunner se = this;
+ rm = new ResourceManager() {
+ @Override
+ protected ApplicationMasterLauncher createAMLauncher() {
+ return new MockAMLauncher(se, this.rmContext, amMap);
+ }
+ };
rm.init(rmConf);
rm.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index d61bf02..5b03d51 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -66,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
@@ -107,11 +109,19 @@ public abstract class AMSimulator extends TaskRunner.Task {
// progress
protected int totalContainers;
protected int finishedContainers;
+
+ // waiting for AM container
+ volatile boolean isAMContainerRunning = false;
+ volatile Container amContainer;
protected final Logger LOG = Logger.getLogger(AMSimulator.class);
-
+
+ // resource for AM container
+ private final static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB = 1024;
+ private final static int MR_AM_CONTAINER_RESOURCE_VCORES = 1;
+
public AMSimulator() {
- this.responseQueue = new LinkedBlockingQueue<AllocateResponse>();
+ this.responseQueue = new LinkedBlockingQueue<>();
}
public void init(int id, int heartbeatInterval,
@@ -142,23 +152,30 @@ public abstract class AMSimulator extends TaskRunner.Task {
// submit application, waiting until ACCEPTED
submitApp();
- // register application master
- registerAM();
-
// track app metrics
trackApp();
}
+ public synchronized void notifyAMContainerLaunched(Container masterContainer)
+ throws Exception {
+ this.amContainer = masterContainer;
+ this.appAttemptId = masterContainer.getId().getApplicationAttemptId();
+ registerAM();
+ isAMContainerRunning = true;
+ }
+
@Override
public void middleStep() throws Exception {
- // process responses in the queue
- processResponseQueue();
-
- // send out request
- sendContainerRequest();
-
- // check whether finish
- checkStop();
+ if (isAMContainerRunning) {
+ // process responses in the queue
+ processResponseQueue();
+
+ // send out request
+ sendContainerRequest();
+
+ // check whether finish
+ checkStop();
+ }
}
@Override
@@ -168,6 +185,22 @@ public abstract class AMSimulator extends TaskRunner.Task {
if (isTracked) {
untrackApp();
}
+
+ // Finish AM container
+ if (amContainer != null) {
+ LOG.info("AM container = " + amContainer.getId() + " reported to finish");
+ se.getNmMap().get(amContainer.getNodeId()).cleanupContainer(
+ amContainer.getId());
+ } else {
+ LOG.info("AM container is null");
+ }
+
+ if (null == appAttemptId) {
+ // If appAttemptId == null, AM is not launched from RM's perspective, so
+ // it's unnecessary to finish am as well
+ return;
+ }
+
// unregister application master
final FinishApplicationMasterRequest finishAMRequest = recordFactory
.newRecordInstance(FinishApplicationMasterRequest.class);
@@ -256,7 +289,9 @@ public abstract class AMSimulator extends TaskRunner.Task {
conLauContext.setLocalResources(new HashMap<String, LocalResource>());
conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
appSubContext.setAMContainerSpec(conLauContext);
- appSubContext.setUnmanagedAM(true);
+ appSubContext.setResource(Resources
+ .createResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB,
+ MR_AM_CONTAINER_RESOURCE_VCORES));
subAppRequest.setApplicationSubmissionContext(appSubContext);
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
ugi.doAs(new PrivilegedExceptionAction<Object>() {
@@ -267,22 +302,6 @@ public abstract class AMSimulator extends TaskRunner.Task {
}
});
LOG.info(MessageFormat.format("Submit a new application {0}", appId));
-
- // waiting until application ACCEPTED
- RMApp app = rm.getRMContext().getRMApps().get(appId);
- while(app.getState() != RMAppState.ACCEPTED) {
- Thread.sleep(10);
- }
-
- // Waiting until application attempt reach LAUNCHED
- // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
- this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
- .getCurrentAppAttempt().getAppAttemptId();
- RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
- .getCurrentAppAttempt();
- while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
- Thread.sleep(10);
- }
}
private void registerAM()
@@ -335,7 +354,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
for (ContainerSimulator cs : csList) {
String rackHostNames[] = SLSUtils.getRackHostName(cs.getHostname());
// check rack local
- String rackname = rackHostNames[0];
+ String rackname = "/" + rackHostNames[0];
if (rackLocalRequestMap.containsKey(rackname)) {
rackLocalRequestMap.get(rackname).setNumContainers(
rackLocalRequestMap.get(rackname).getNumContainers() + 1);
@@ -383,4 +402,12 @@ public abstract class AMSimulator extends TaskRunner.Task {
public int getNumTasks() {
return totalContainers;
}
+
+ public ApplicationId getApplicationId() {
+ return appId;
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return appAttemptId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
index da267a1..e726b09 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
@@ -27,6 +27,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.avro.Protocol;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
@@ -63,10 +64,10 @@ public class MRAMSimulator extends AMSimulator {
private static final int PRIORITY_REDUCE = 10;
private static final int PRIORITY_MAP = 20;
-
+
// pending maps
private LinkedList<ContainerSimulator> pendingMaps =
- new LinkedList<ContainerSimulator>();
+ new LinkedList<>();
// pending failed maps
private LinkedList<ContainerSimulator> pendingFailedMaps =
@@ -107,14 +108,9 @@ public class MRAMSimulator extends AMSimulator {
private int mapTotal = 0;
private int reduceFinished = 0;
private int reduceTotal = 0;
- // waiting for AM container
- private boolean isAMContainerRunning = false;
- private Container amContainer;
+
// finished
private boolean isFinished = false;
- // resource for AM container
- private final static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB = 1024;
- private final static int MR_AM_CONTAINER_RESOURCE_VCORES = 1;
public final Logger LOG = Logger.getLogger(MRAMSimulator.class);
@@ -131,83 +127,34 @@ public class MRAMSimulator extends AMSimulator {
for (ContainerSimulator cs : containerList) {
if (cs.getType().equals("map")) {
cs.setPriority(PRIORITY_MAP);
- pendingMaps.add(cs);
+ allMaps.add(cs);
} else if (cs.getType().equals("reduce")) {
cs.setPriority(PRIORITY_REDUCE);
- pendingReduces.add(cs);
+ allReduces.add(cs);
}
}
- allMaps.addAll(pendingMaps);
- allReduces.addAll(pendingReduces);
- mapTotal = pendingMaps.size();
- reduceTotal = pendingReduces.size();
+
+ LOG.info(MessageFormat
+ .format("Added new job with {0} mapper and {1} reducers",
+ allMaps.size(), allReduces.size()));
+
+ mapTotal = allMaps.size();
+ reduceTotal = allReduces.size();
totalContainers = mapTotal + reduceTotal;
}
@Override
- public void firstStep() throws Exception {
- super.firstStep();
-
- requestAMContainer();
- }
-
- /**
- * send out request for AM container
- */
- protected void requestAMContainer()
- throws YarnException, IOException, InterruptedException {
- List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
- ResourceRequest amRequest = createResourceRequest(
- BuilderUtils.newResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB,
- MR_AM_CONTAINER_RESOURCE_VCORES),
- ResourceRequest.ANY, 1, 1);
- ask.add(amRequest);
- LOG.debug(MessageFormat.format("Application {0} sends out allocate " +
- "request for its AM", appId));
- final AllocateRequest request = this.createAllocateRequest(ask);
-
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser(appAttemptId.toString());
- Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps()
- .get(appAttemptId.getApplicationId())
- .getRMAppAttempt(appAttemptId).getAMRMToken();
- ugi.addTokenIdentifier(token.decodeIdentifier());
- AllocateResponse response = ugi.doAs(
- new PrivilegedExceptionAction<AllocateResponse>() {
- @Override
- public AllocateResponse run() throws Exception {
- return rm.getApplicationMasterService().allocate(request);
- }
- });
- if (response != null) {
- responseQueue.put(response);
+ public synchronized void notifyAMContainerLaunched(Container masterContainer)
+ throws Exception {
+ if (null != masterContainer) {
+ restart();
+ super.notifyAMContainerLaunched(masterContainer);
}
}
@Override
@SuppressWarnings("unchecked")
- protected void processResponseQueue()
- throws InterruptedException, YarnException, IOException {
- // Check whether receive the am container
- if (!isAMContainerRunning) {
- if (!responseQueue.isEmpty()) {
- AllocateResponse response = responseQueue.take();
- if (response != null
- && !response.getAllocatedContainers().isEmpty()) {
- // Get AM container
- Container container = response.getAllocatedContainers().get(0);
- se.getNmMap().get(container.getNodeId())
- .addNewContainer(container, -1L);
- // Start AM container
- amContainer = container;
- LOG.debug(MessageFormat.format("Application {0} starts its " +
- "AM container ({1}).", appId, amContainer.getId()));
- isAMContainerRunning = true;
- }
- }
- return;
- }
-
+ protected void processResponseQueue() throws Exception {
while (! responseQueue.isEmpty()) {
AllocateResponse response = responseQueue.take();
@@ -228,12 +175,16 @@ public class MRAMSimulator extends AMSimulator {
assignedReduces.remove(containerId);
reduceFinished ++;
finishedContainers ++;
- } else {
+ } else if (amContainer.getId().equals(containerId)){
// am container released event
isFinished = true;
LOG.info(MessageFormat.format("Application {0} goes to " +
"finish.", appId));
}
+
+ if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
+ lastStep();
+ }
} else {
// container to be killed
if (assignedMaps.containsKey(containerId)) {
@@ -244,10 +195,9 @@ public class MRAMSimulator extends AMSimulator {
LOG.debug(MessageFormat.format("Application {0} has one " +
"reducer killed ({1}).", appId, containerId));
pendingFailedReduces.add(assignedReduces.remove(containerId));
- } else {
+ } else if (amContainer.getId().equals(containerId)){
LOG.info(MessageFormat.format("Application {0}'s AM is " +
- "going to be killed. Restarting...", appId));
- restart();
+ "going to be killed. Waiting for rescheduling...", appId));
}
}
}
@@ -255,11 +205,8 @@ public class MRAMSimulator extends AMSimulator {
// check finished
if (isAMContainerRunning &&
- (mapFinished == mapTotal) &&
- (reduceFinished == reduceTotal)) {
- // to release the AM container
- se.getNmMap().get(amContainer.getNodeId())
- .cleanupContainer(amContainer.getId());
+ (mapFinished >= mapTotal) &&
+ (reduceFinished >= reduceTotal)) {
isAMContainerRunning = false;
LOG.debug(MessageFormat.format("Application {0} sends out event " +
"to clean up its AM container.", appId));
@@ -293,21 +240,38 @@ public class MRAMSimulator extends AMSimulator {
*/
private void restart()
throws YarnException, IOException, InterruptedException {
- // clear
- finishedContainers = 0;
+ // clear
isFinished = false;
- mapFinished = 0;
- reduceFinished = 0;
pendingFailedMaps.clear();
pendingMaps.clear();
pendingReduces.clear();
pendingFailedReduces.clear();
- pendingMaps.addAll(allMaps);
- pendingReduces.addAll(pendingReduces);
- isAMContainerRunning = false;
+
+ // Only add totalMaps - finishedMaps
+ int added = 0;
+ for (ContainerSimulator cs : allMaps) {
+ if (added >= mapTotal - mapFinished) {
+ break;
+ }
+ pendingMaps.add(cs);
+ }
+
+ // And same, only add totalReduces - finishedReduces
+ added = 0;
+ for (ContainerSimulator cs : allReduces) {
+ if (added >= reduceTotal - reduceFinished) {
+ break;
+ }
+ pendingReduces.add(cs);
+ }
amContainer = null;
- // resent am container request
- requestAMContainer();
+ }
+
+ private List<ContainerSimulator> mergeLists(List<ContainerSimulator> left, List<ContainerSimulator> right) {
+ List<ContainerSimulator> list = new ArrayList<>();
+ list.addAll(left);
+ list.addAll(right);
+ return list;
}
@Override
@@ -319,44 +283,48 @@ public class MRAMSimulator extends AMSimulator {
// send out request
List<ResourceRequest> ask = null;
- if (isAMContainerRunning) {
- if (mapFinished != mapTotal) {
- // map phase
- if (! pendingMaps.isEmpty()) {
- ask = packageRequests(pendingMaps, PRIORITY_MAP);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "request for {1} mappers.", appId, pendingMaps.size()));
- scheduledMaps.addAll(pendingMaps);
- pendingMaps.clear();
- } else if (! pendingFailedMaps.isEmpty() && scheduledMaps.isEmpty()) {
- ask = packageRequests(pendingFailedMaps, PRIORITY_MAP);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "requests for {1} failed mappers.", appId,
- pendingFailedMaps.size()));
- scheduledMaps.addAll(pendingFailedMaps);
- pendingFailedMaps.clear();
- }
- } else if (reduceFinished != reduceTotal) {
- // reduce phase
- if (! pendingReduces.isEmpty()) {
- ask = packageRequests(pendingReduces, PRIORITY_REDUCE);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "requests for {1} reducers.", appId, pendingReduces.size()));
- scheduledReduces.addAll(pendingReduces);
- pendingReduces.clear();
- } else if (! pendingFailedReduces.isEmpty()
- && scheduledReduces.isEmpty()) {
- ask = packageRequests(pendingFailedReduces, PRIORITY_REDUCE);
- LOG.debug(MessageFormat.format("Application {0} sends out " +
- "request for {1} failed reducers.", appId,
- pendingFailedReduces.size()));
- scheduledReduces.addAll(pendingFailedReduces);
- pendingFailedReduces.clear();
- }
+ if (mapFinished != mapTotal) {
+ // map phase
+ if (!pendingMaps.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingMaps, scheduledMaps),
+ PRIORITY_MAP);
+ LOG.debug(MessageFormat
+ .format("Application {0} sends out " + "request for {1} mappers.",
+ appId, pendingMaps.size()));
+ scheduledMaps.addAll(pendingMaps);
+ pendingMaps.clear();
+ } else if (!pendingFailedMaps.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingFailedMaps, scheduledMaps),
+ PRIORITY_MAP);
+ LOG.debug(MessageFormat.format(
+ "Application {0} sends out " + "requests for {1} failed mappers.",
+ appId, pendingFailedMaps.size()));
+ scheduledMaps.addAll(pendingFailedMaps);
+ pendingFailedMaps.clear();
+ }
+ } else if (reduceFinished != reduceTotal) {
+ // reduce phase
+ if (!pendingReduces.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingReduces, scheduledReduces),
+ PRIORITY_REDUCE);
+ LOG.debug(MessageFormat
+ .format("Application {0} sends out " + "requests for {1} reducers.",
+ appId, pendingReduces.size()));
+ scheduledReduces.addAll(pendingReduces);
+ pendingReduces.clear();
+ } else if (!pendingFailedReduces.isEmpty()) {
+ ask = packageRequests(mergeLists(pendingFailedReduces, scheduledReduces),
+ PRIORITY_REDUCE);
+ LOG.debug(MessageFormat.format(
+ "Application {0} sends out " + "request for {1} failed reducers.",
+ appId, pendingFailedReduces.size()));
+ scheduledReduces.addAll(pendingFailedReduces);
+ pendingFailedReduces.clear();
}
}
+
if (ask == null) {
- ask = new ArrayList<ResourceRequest>();
+ ask = new ArrayList<>();
}
final AllocateRequest request = createAllocateRequest(ask);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
new file mode 100644
index 0000000..20cf3e5
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls.resourcemanager;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
+
+import java.util.Map;
+
+public class MockAMLauncher extends ApplicationMasterLauncher
+ implements EventHandler<AMLauncherEvent> {
+ private static final Log LOG = LogFactory.getLog(
+ MockAMLauncher.class);
+
+ Map<String, AMSimulator> amMap;
+ SLSRunner se;
+
+ public MockAMLauncher(SLSRunner se, RMContext rmContext,
+ Map<String, AMSimulator> amMap) {
+ super(rmContext);
+ this.amMap = amMap;
+ this.se = se;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ // Do nothing
+ }
+
+ private void setupAMRMToken(RMAppAttempt appAttempt) {
+ // Setup AMRMToken
+ Token<AMRMTokenIdentifier> amrmToken =
+ super.context.getAMRMTokenSecretManager().createAndGetAMRMToken(
+ appAttempt.getAppAttemptId());
+ ((RMAppAttemptImpl) appAttempt).setAMRMToken(amrmToken);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void handle(AMLauncherEvent event) {
+ if (AMLauncherEventType.LAUNCH == event.getType()) {
+ ApplicationId appId =
+ event.getAppAttempt().getAppAttemptId().getApplicationId();
+
+ // find AMSimulator
+ for (AMSimulator ams : amMap.values()) {
+ if (ams.getApplicationId() != null && ams.getApplicationId().equals(
+ appId)) {
+ try {
+ Container amContainer = event.getAppAttempt().getMasterContainer();
+
+ setupAMRMToken(event.getAppAttempt());
+
+ // Notify RMAppAttempt to change state
+ super.context.getDispatcher().getEventHandler().handle(
+ new RMAppAttemptEvent(event.getAppAttempt().getAppAttemptId(),
+ RMAppAttemptEventType.LAUNCHED));
+
+ ams.notifyAMContainerLaunched(
+ event.getAppAttempt().getMasterContainer());
+ LOG.info("Notify AM launcher launched:" + amContainer.getId());
+
+ se.getNmMap().get(amContainer.getNodeId())
+ .addNewContainer(amContainer, 100000000L);
+
+ return;
+ } catch (Exception e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+ }
+
+ throw new YarnRuntimeException(
+ "Didn't find any AMSimulator for applicationId=" + appId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ffa27/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index 8388273..cd4377e 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -556,6 +556,30 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
}
}
);
+ metrics.register("variable.cluster.reserved.memory",
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ if(getRootQueueMetrics() == null) {
+ return 0L;
+ } else {
+ return getRootQueueMetrics().getReservedMB();
+ }
+ }
+ }
+ );
+ metrics.register("variable.cluster.reserved.vcores",
+ new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ if(getRootQueueMetrics() == null) {
+ return 0;
+ } else {
+ return getRootQueueMetrics().getReservedVirtualCores();
+ }
+ }
+ }
+ );
}
private void registerContainerAppNumMetrics() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-14114 S3A can no longer
handle unencoded + in URIs. Contributed by Sean Mackrory.
Posted by su...@apache.org.
HADOOP-14114 S3A can no longer handle unencoded + in URIs. Contributed by Sean Mackrory.
(cherry picked from commit ff87ca84418a710c6dc884fe8c70947fcc6489d5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c22a916
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c22a916
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c22a916
Branch: refs/heads/YARN-2915
Commit: 9c22a91662af24569191ce45289ef8266e8755cc
Parents: 132f758
Author: Steve Loughran <st...@apache.org>
Authored: Fri Feb 24 10:41:36 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Feb 24 10:41:36 2017 +0000
----------------------------------------------------------------------
.../hadoop/fs/s3native/S3xLoginHelper.java | 15 ++++++++++-
.../hadoop/fs/s3native/TestS3xLoginHelper.java | 28 ++++++++++++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c22a916/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index 97ece37..862ce6b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -48,6 +48,13 @@ public final class S3xLoginHelper {
"The Filesystem URI contains login details."
+" This is insecure and may be unsupported in future.";
+ public static final String PLUS_WARNING =
+ "Secret key contains a special character that should be URL encoded! " +
+ "Attempting to resolve...";
+
+ public static final String PLUS_UNENCODED = "+";
+ public static final String PLUS_ENCODED = "%2B";
+
/**
* Build the filesystem URI. This can include stripping down of part
* of the URI.
@@ -112,7 +119,13 @@ public final class S3xLoginHelper {
int loginSplit = login.indexOf(':');
if (loginSplit > 0) {
String user = login.substring(0, loginSplit);
- String password = URLDecoder.decode(login.substring(loginSplit + 1),
+ String encodedPassword = login.substring(loginSplit + 1);
+ if (encodedPassword.contains(PLUS_UNENCODED)) {
+ LOG.warn(PLUS_WARNING);
+ encodedPassword = encodedPassword.replaceAll("\\" + PLUS_UNENCODED,
+ PLUS_ENCODED);
+ }
+ String password = URLDecoder.decode(encodedPassword,
"UTF-8");
return new Login(user, password);
} else if (loginSplit == 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c22a916/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
index bd2ac1e..3761cb7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
@@ -32,9 +32,13 @@ public class TestS3xLoginHelper extends Assert {
public static final String BUCKET = "s3a://bucket";
private static final URI ENDPOINT = uri(BUCKET);
public static final String S = "%2f";
+ public static final String P = "%2b";
+ public static final String P_RAW = "+";
public static final String USER = "user";
public static final String PASS = "pass";
public static final String PASLASHSLASH = "pa" + S + S;
+ public static final String PAPLUS = "pa" + P;
+ public static final String PAPLUS_RAW = "pa" + P_RAW;
public static final URI WITH_USER_AND_PASS = uri("s3a://user:pass@bucket");
public static final Path PATH_WITH_LOGIN =
@@ -42,6 +46,10 @@ public class TestS3xLoginHelper extends Assert {
public static final URI WITH_SLASH_IN_PASS = uri(
"s3a://user:" + PASLASHSLASH + "@bucket");
+ public static final URI WITH_PLUS_IN_PASS = uri(
+ "s3a://user:" + PAPLUS + "@bucket");
+ public static final URI WITH_PLUS_RAW_IN_PASS = uri(
+ "s3a://user:" + PAPLUS_RAW + "@bucket");
public static final URI USER_NO_PASS = uri("s3a://user@bucket");
public static final URI WITH_USER_AND_COLON = uri("s3a://user:@bucket");
public static final URI NO_USER = uri("s3a://:pass@bucket");
@@ -117,6 +125,16 @@ public class TestS3xLoginHelper extends Assert {
}
@Test
+ public void testLoginWithPlusInPass() throws Throwable {
+ assertMatchesLogin(USER, "pa+", WITH_PLUS_IN_PASS);
+ }
+
+ @Test
+ public void testLoginWithPlusRawInPass() throws Throwable {
+ assertMatchesLogin(USER, "pa+", WITH_PLUS_RAW_IN_PASS);
+ }
+
+ @Test
public void testLoginWithUser() throws Throwable {
assertMatchesLogin(USER, "", USER_NO_PASS);
}
@@ -152,6 +170,16 @@ public class TestS3xLoginHelper extends Assert {
}
@Test
+ public void testFsUriWithPlusInPass() throws Throwable {
+ assertMatchesEndpoint(WITH_PLUS_IN_PASS);
+ }
+
+ @Test
+ public void testFsUriWithPlusRawInPass() throws Throwable {
+ assertMatchesEndpoint(WITH_PLUS_RAW_IN_PASS);
+ }
+
+ @Test
public void testFsUriWithUser() throws Throwable {
assertMatchesEndpoint(USER_NO_PASS);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-5872. Add AlwayReject
policies for router and amrmproxy. (Carlo Curino via Subru).
Posted by su...@apache.org.
YARN-5872. Add AlwayReject policies for router and amrmproxy. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4ea2bb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4ea2bb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4ea2bb2
Branch: refs/heads/YARN-2915
Commit: f4ea2bb298013de39c440aeccb096ecb59fa8d54
Parents: eb7a7a7
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Nov 22 18:37:30 2016 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
.../amrmproxy/RejectAMRMProxyPolicy.java | 67 +++++++++++++++++
.../manager/RejectAllPolicyManager.java | 40 ++++++++++
.../policies/router/RejectRouterPolicy.java | 66 +++++++++++++++++
.../amrmproxy/TestRejectAMRMProxyPolicy.java | 78 ++++++++++++++++++++
.../manager/TestRejectAllPolicyManager.java | 40 ++++++++++
.../policies/router/TestRejectRouterPolicy.java | 63 ++++++++++++++++
6 files changed, 354 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ea2bb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
new file mode 100644
index 0000000..3783df6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} that simply
+ * rejects all requests. Useful to prevent apps from accessing any sub-cluster.
+ */
+public class RejectAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ private Set<SubClusterId> knownClusterIds = new HashSet<>();
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+ // overrides initialize to avoid weight checks that do no apply for
+ // this policy.
+ FederationPolicyInitializationContextValidator.validate(policyContext,
+ this.getClass().getCanonicalName());
+ setPolicyContext(policyContext);
+ }
+
+ @Override
+ public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
+ List<ResourceRequest> resourceRequests) throws YarnException {
+ throw new FederationPolicyException("The policy configured for this queue "
+ + "rejects all routing requests by construction.");
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+ // This might be invoked for applications started with a previous policy,
+ // do nothing for this policy.
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ea2bb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java
new file mode 100644
index 0000000..7bd7a1b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/manager/RejectAllPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.RejectAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.RejectRouterPolicy;
+
+/**
+ * This class represents a simple implementation of a {@code
+ * FederationPolicyManager}.
+ *
+ * This policy rejects all reuqests for both router and amrmproxy routing. This
+ * is to be used to prevent applications in a specific queue (or if used as
+ * default for non-configured queues) from accessing cluster resources.
+ */
+public class RejectAllPolicyManager extends AbstractPolicyManager {
+
+ public RejectAllPolicyManager() {
+ // this structurally hard-codes two compatible policies for Router and
+ // AMRMProxy.
+ routerFederationPolicy = RejectRouterPolicy.class;
+ amrmProxyFederationPolicy = RejectAMRMProxyPolicy.class;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ea2bb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
new file mode 100644
index 0000000..faf3279
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * This {@link FederationRouterPolicy} simply rejects all incoming requests.
+ * This is useful to prevent applications running in a queue to be run
+ * anywhere in the federated cluster.
+ */
+public class RejectRouterPolicy extends AbstractRouterPolicy {
+
+ @Override
+ public void reinitialize(
+ FederationPolicyInitializationContext federationPolicyContext)
+ throws FederationPolicyInitializationException {
+ FederationPolicyInitializationContextValidator
+ .validate(federationPolicyContext, this.getClass().getCanonicalName());
+ setPolicyContext(federationPolicyContext);
+ }
+
+ /**
+ * The policy always reject requests.
+ *
+ * @param appSubmissionContext the context for the app being submitted.
+ *
+ * @return (never).
+ *
+ * @throws YarnException (always) to prevent applications in this queue to
+ * be run anywhere in the federated cluster.
+ */
+ public SubClusterId getHomeSubcluster(
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
+
+ // run standard validation, as error might differ
+ validate(appSubmissionContext);
+
+ throw new FederationPolicyException("The policy configured for this queue"
+ + " (" + appSubmissionContext.getQueue() + ") reject all routing "
+ + "requests by construction. Application " + appSubmissionContext
+ .getApplicationId() + " cannot be routed to any RM.");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ea2bb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
new file mode 100644
index 0000000..41e7fed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link RejectAMRMProxyPolicy}.
+ */
+public class TestRejectAMRMProxyPolicy
+ extends BaseFederationPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new RejectAMRMProxyPolicy());
+ // needed for base test to work
+ setPolicyInfo(mock(WeightedPolicyInfo.class));
+
+ for (int i = 1; i <= 2; i++) {
+ SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
+ SubClusterInfo sci = mock(SubClusterInfo.class);
+ when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
+ when(sci.getSubClusterId()).thenReturn(sc.toId());
+ getActiveSubclusters().put(sc.toId(), sci);
+ }
+
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ mock(WeightedPolicyInfo.class), getActiveSubclusters());
+
+ }
+
+ @Test (expected = FederationPolicyException.class)
+ public void testSplitAllocateRequest() throws Exception {
+ // verify the request is broadcasted to all subclusters
+ String[] hosts = new String[] {"host1", "host2" };
+ List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
+ .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
+
+ Map<SubClusterId, List<ResourceRequest>> response =
+ ((FederationAMRMProxyPolicy) getPolicy())
+ .splitResourceRequests(resourceRequests);
+ }
+
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ea2bb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java
new file mode 100644
index 0000000..e4dc7f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/manager/TestRejectAllPolicyManager.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.manager;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.RejectAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.router.RejectRouterPolicy;
+import org.junit.Before;
+
+/**
+ * Simple test of {@link RejectAllPolicyManager}.
+ */
+public class TestRejectAllPolicyManager extends BasePolicyManagerTest {
+
+ @Before
+ public void setup() {
+ // config policy
+ wfp = new RejectAllPolicyManager();
+ wfp.setQueue("queue1");
+
+ // set expected params that the base test class will use for tests
+ expectedPolicyManager = RejectAllPolicyManager.class;
+ expectedAMRMProxyPolicy = RejectAMRMProxyPolicy.class;
+ expectedRouterPolicy = RejectRouterPolicy.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ea2bb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
new file mode 100644
index 0000000..049ebbf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.router;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Simple test class for the {@link RejectRouterPolicy}. Tests that one of the
+ * active subcluster is chosen.
+ */
+public class TestRejectRouterPolicy extends BaseRouterPoliciesTest {
+
+ @Before
+ public void setUp() throws Exception {
+ setPolicy(new RejectRouterPolicy());
+
+ // setting up the active sub-clusters for this test
+ setMockActiveSubclusters(2);
+
+ // initialize policy with context
+ FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
+ getPolicyInfo(), getActiveSubclusters());
+
+ }
+
+ @Test(expected = FederationPolicyException.class)
+ public void testNoClusterIsChosen() throws YarnException {
+ ((FederationRouterPolicy) getPolicy())
+ .getHomeSubcluster(getApplicationSubmissionContext());
+ }
+
+ @Override
+ @Test(expected = FederationPolicyException.class)
+ public void testNullQueueRouting() throws YarnException {
+ FederationRouterPolicy localPolicy = (FederationRouterPolicy) getPolicy();
+ ApplicationSubmissionContext applicationSubmissionContext =
+ ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+ false, false, 0, Resources.none(), null, false, null, null);
+ localPolicy.getHomeSubcluster(applicationSubmissionContext);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDFS-4025. QJM: Sychronize past
log segments to JNs that missed them. Contributed by Hanisha Koneru.
Posted by su...@apache.org.
HDFS-4025. QJM: Sychronize past log segments to JNs that missed them. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13d4bcfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13d4bcfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13d4bcfe
Branch: refs/heads/YARN-2915
Commit: 13d4bcfe3535a2df79c2a56e7578716d15497ff4
Parents: b10e962
Author: Jing Zhao <ji...@apache.org>
Authored: Wed Feb 22 16:33:38 2017 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Wed Feb 22 16:33:38 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +
.../qjournal/client/QuorumJournalManager.java | 38 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 9 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 19 +
.../hdfs/qjournal/server/JournalNode.java | 23 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 413 +++++++++++++++++++
.../hadoop/hdfs/server/common/Storage.java | 9 +
.../apache/hadoop/hdfs/server/common/Util.java | 46 ++-
.../hadoop/hdfs/server/namenode/NNStorage.java | 5 +-
.../hdfs/server/namenode/TransferFsImage.java | 3 +-
.../src/main/resources/hdfs-default.xml | 41 ++
.../hdfs/qjournal/MiniJournalCluster.java | 8 +
.../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 1 +
.../hdfs/qjournal/TestJournalNodeSync.java | 264 ++++++++++++
14 files changed, 853 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index cf1d21a..cfd16aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -707,6 +707,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY = "dfs.image.transfer.chunksize";
public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024;
+ // Edit Log segment transfer timeout
+ public static final String DFS_EDIT_LOG_TRANSFER_TIMEOUT_KEY =
+ "dfs.edit.log.transfer.timeout";
+ public static final int DFS_EDIT_LOG_TRANSFER_TIMEOUT_DEFAULT = 30 * 1000;
+
+ // Throttling Edit Log Segment transfer for Journal Sync
+ public static final String DFS_EDIT_LOG_TRANSFER_RATE_KEY =
+ "dfs.edit.log.transfer.bandwidthPerSec";
+ public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling
+
// Datanode File IO Stats
public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY =
"dfs.datanode.enable.fileio.profiling";
@@ -891,6 +901,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_JOURNALNODE_KEYTAB_FILE_KEY = "dfs.journalnode.keytab.file";
public static final String DFS_JOURNALNODE_KERBEROS_PRINCIPAL_KEY = "dfs.journalnode.kerberos.principal";
public static final String DFS_JOURNALNODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY = "dfs.journalnode.kerberos.internal.spnego.principal";
+ public static final String DFS_JOURNALNODE_ENABLE_SYNC_KEY =
+ "dfs.journalnode.enable.sync";
+ public static final boolean DFS_JOURNALNODE_ENABLE_SYNC_DEFAULT = false;
+ public static final String DFS_JOURNALNODE_SYNC_INTERVAL_KEY =
+ "dfs.journalnode.sync.interval";
+ public static final long DFS_JOURNALNODE_SYNC_INTERVAL_DEFAULT = 2*60*1000L;
// Journal-node related configs for the client side.
public static final String DFS_QJOURNAL_QUEUE_SIZE_LIMIT_KEY = "dfs.qjournal.queued-edits.limit.mb";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index ae3358b..97c0050 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
-import java.net.UnknownHostException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -42,6 +41,7 @@ import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRe
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
@@ -51,8 +51,6 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.StringUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
@@ -362,41 +360,17 @@ public class QuorumJournalManager implements JournalManager {
URI uri, NamespaceInfo nsInfo, AsyncLogger.Factory factory)
throws IOException {
List<AsyncLogger> ret = Lists.newArrayList();
- List<InetSocketAddress> addrs = getLoggerAddresses(uri);
+ List<InetSocketAddress> addrs = Util.getAddressesList(uri);
+ if (addrs.size() % 2 == 0) {
+ LOG.warn("Quorum journal URI '" + uri + "' has an even number " +
+ "of Journal Nodes specified. This is not recommended!");
+ }
String jid = parseJournalId(uri);
for (InetSocketAddress addr : addrs) {
ret.add(factory.createLogger(conf, nsInfo, jid, addr));
}
return ret;
}
-
- private static List<InetSocketAddress> getLoggerAddresses(URI uri)
- throws IOException {
- String authority = uri.getAuthority();
- Preconditions.checkArgument(authority != null && !authority.isEmpty(),
- "URI has no authority: " + uri);
-
- String[] parts = StringUtils.split(authority, ';');
- for (int i = 0; i < parts.length; i++) {
- parts[i] = parts[i].trim();
- }
-
- if (parts.length % 2 == 0) {
- LOG.warn("Quorum journal URI '" + uri + "' has an even number " +
- "of Journal Nodes specified. This is not recommended!");
- }
-
- List<InetSocketAddress> addrs = Lists.newArrayList();
- for (String addr : parts) {
- InetSocketAddress isa = NetUtils.createSocketAddr(
- addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT);
- if (isa.isUnresolved()) {
- throw new UnknownHostException(addr);
- }
- addrs.add(isa);
- }
- return addrs;
- }
@Override
public EditLogOutputStream startLogSegment(long txId, int layoutVersion)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
index 07c9286..8f40f6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
@@ -49,7 +49,6 @@ class JNStorage extends Storage {
private final FileJournalManager fjm;
private final StorageDirectory sd;
private StorageState state;
-
private static final List<Pattern> CURRENT_DIR_PURGE_REGEXES =
ImmutableList.of(
@@ -121,6 +120,14 @@ class JNStorage extends Storage {
return new File(sd.getCurrentDir(), name);
}
+ File getTemporaryEditsFile(long startTxId, long endTxId, long timestamp) {
+ return NNStorage.getTemporaryEditsFile(sd, startTxId, endTxId, timestamp);
+ }
+
+ File getFinalizedEditsFile(long startTxId, long endTxId) {
+ return NNStorage.getFinalizedEditsFile(sd, startTxId, endTxId);
+ }
+
/**
* @return the path for the file which contains persisted data for the
* paxos-like recovery process for the given log segment.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index 3760641..ca21373 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -1092,6 +1092,25 @@ public class Journal implements Closeable {
committedTxnId.set(startTxId - 1);
}
+ synchronized boolean renameTmpSegment(File tmpFile, File finalFile,
+ long endTxId) throws IOException {
+ final boolean success;
+ if (endTxId <= committedTxnId.get()) {
+ success = tmpFile.renameTo(finalFile);
+ if (!success) {
+ LOG.warn("Unable to rename edits file from " + tmpFile + " to " +
+ finalFile);
+ }
+ } else {
+ success = false;
+ LOG.error("The endTxId of the temporary file is not less than the " +
+ "last committed transaction id. Aborting renaming to final file" +
+ finalFile);
+ }
+
+ return success;
+ }
+
public Long getJournalCTime() throws IOException {
return storage.getJournalManager().getJournalCTime();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
index cde0112..42e9be7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
@@ -68,6 +68,8 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
private JournalNodeRpcServer rpcServer;
private JournalNodeHttpServer httpServer;
private final Map<String, Journal> journalsById = Maps.newHashMap();
+ private final Map<String, JournalNodeSyncer> journalSyncersById = Maps
+ .newHashMap();
private ObjectName journalNodeInfoBeanName;
private String httpServerURI;
private File localDir;
@@ -92,11 +94,24 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
LOG.info("Initializing journal in directory " + logDir);
journal = new Journal(conf, logDir, jid, startOpt, new ErrorReporter());
journalsById.put(jid, journal);
+
+ // Start SyncJouranl thread, if JournalNode Sync is enabled
+ if (conf.getBoolean(
+ DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_KEY,
+ DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_DEFAULT)) {
+ startSyncer(journal, jid);
+ }
}
-
+
return journal;
}
+ private void startSyncer(Journal journal, String jid) {
+ JournalNodeSyncer jSyncer = new JournalNodeSyncer(this, journal, jid, conf);
+ journalSyncersById.put(jid, jSyncer);
+ jSyncer.start();
+ }
+
@VisibleForTesting
public Journal getOrCreateJournal(String jid) throws IOException {
return getOrCreateJournal(jid, StartupOption.REGULAR);
@@ -190,7 +205,11 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
*/
public void stop(int rc) {
this.resultCode = rc;
-
+
+ for (JournalNodeSyncer jSyncer : journalSyncersById.values()) {
+ jSyncer.stopSync();
+ }
+
if (rpcServer != null) {
rpcServer.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
new file mode 100644
index 0000000..f195c00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
@@ -0,0 +1,413 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.qjournal.server;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos
+ .JournalIdProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos
+ .GetEditLogManifestRequestProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos
+ .GetEditLogManifestResponseProto;
+import org.apache.hadoop.hdfs.qjournal.protocolPB.QJournalProtocolPB;
+import org.apache.hadoop.hdfs.server.common.Util;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
+import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+
+/**
+ * A Journal Sync thread runs through the lifetime of the JN. It periodically
+ * gossips with other journal nodes to compare edit log manifests and if it
+ * detects any missing log segment, it downloads it from the other journal node
+ */
+@InterfaceAudience.Private
+public class JournalNodeSyncer {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ JournalNodeSyncer.class);
+ private final JournalNode jn;
+ private final Journal journal;
+ private final String jid;
+ private final JournalIdProto jidProto;
+ private final JNStorage jnStorage;
+ private final Configuration conf;
+ private volatile Daemon syncJournalDaemon;
+ private volatile boolean shouldSync = true;
+
+ private List<JournalNodeProxy> otherJNProxies = Lists.newArrayList();
+ private int numOtherJNs;
+ private int journalNodeIndexForSync = 0;
+ private final long journalSyncInterval;
+ private final int logSegmentTransferTimeout;
+ private final DataTransferThrottler throttler;
+
+ JournalNodeSyncer(JournalNode jouranlNode, Journal journal, String jid,
+ Configuration conf) {
+ this.jn = jouranlNode;
+ this.journal = journal;
+ this.jid = jid;
+ this.jidProto = convertJournalId(this.jid);
+ this.jnStorage = journal.getStorage();
+ this.conf = conf;
+ journalSyncInterval = conf.getLong(
+ DFSConfigKeys.DFS_JOURNALNODE_SYNC_INTERVAL_KEY,
+ DFSConfigKeys.DFS_JOURNALNODE_SYNC_INTERVAL_DEFAULT);
+ logSegmentTransferTimeout = conf.getInt(
+ DFSConfigKeys.DFS_EDIT_LOG_TRANSFER_TIMEOUT_KEY,
+ DFSConfigKeys.DFS_EDIT_LOG_TRANSFER_TIMEOUT_DEFAULT);
+ throttler = getThrottler(conf);
+ }
+
+ void stopSync() {
+ shouldSync = false;
+ if (syncJournalDaemon != null) {
+ syncJournalDaemon.interrupt();
+ }
+ }
+
+ public void start() {
+ LOG.info("Starting SyncJournal daemon for journal " + jid);
+ if (getOtherJournalNodeProxies()) {
+ startSyncJournalsDaemon();
+ } else {
+ LOG.warn("Failed to start SyncJournal daemon for journal " + jid);
+ }
+ }
+
+ private boolean getOtherJournalNodeProxies() {
+ List<InetSocketAddress> otherJournalNodes = getOtherJournalNodeAddrs();
+ if (otherJournalNodes == null || otherJournalNodes.isEmpty()) {
+ LOG.warn("Other JournalNode addresses not available. Journal Syncing " +
+ "cannot be done");
+ return false;
+ }
+ for (InetSocketAddress addr : otherJournalNodes) {
+ try {
+ otherJNProxies.add(new JournalNodeProxy(addr));
+ } catch (IOException e) {
+ LOG.warn("Could not add proxy for Journal at addresss " + addr, e);
+ }
+ }
+ if (otherJNProxies.isEmpty()) {
+ LOG.error("Cannot sync as there is no other JN available for sync.");
+ return false;
+ }
+ numOtherJNs = otherJNProxies.size();
+ return true;
+ }
+
+ private void startSyncJournalsDaemon() {
+ syncJournalDaemon = new Daemon(new Runnable() {
+ @Override
+ public void run() {
+ while(shouldSync) {
+ try {
+ if (!journal.isFormatted()) {
+ LOG.warn("Journal not formatted. Cannot sync.");
+ } else {
+ syncJournals();
+ }
+ Thread.sleep(journalSyncInterval);
+ } catch (Throwable t) {
+ if (!shouldSync) {
+ if (t instanceof InterruptedException) {
+ LOG.info("Stopping JournalNode Sync.");
+ } else {
+ LOG.warn("JournalNodeSyncer received an exception while " +
+ "shutting down.", t);
+ }
+ break;
+ } else {
+ if (t instanceof InterruptedException) {
+ LOG.warn("JournalNodeSyncer interrupted", t);
+ break;
+ }
+ }
+ LOG.error(
+ "JournalNodeSyncer daemon received Runtime exception. ", t);
+ }
+ }
+ }
+ });
+ syncJournalDaemon.start();
+ }
+
+ private void syncJournals() {
+ syncWithJournalAtIndex(journalNodeIndexForSync);
+ journalNodeIndexForSync = (journalNodeIndexForSync + 1) % numOtherJNs;
+ }
+
+ private void syncWithJournalAtIndex(int index) {
+ LOG.info("Syncing Journal " + jn.getBoundIpcAddress().getAddress() + ":"
+ + jn.getBoundIpcAddress().getPort() + " with "
+ + otherJNProxies.get(index) + ", journal id: " + jid);
+ final QJournalProtocolPB jnProxy = otherJNProxies.get(index).jnProxy;
+ if (jnProxy == null) {
+ LOG.error("JournalNode Proxy not found.");
+ return;
+ }
+
+ List<RemoteEditLog> thisJournalEditLogs;
+ try {
+ thisJournalEditLogs = journal.getEditLogManifest(0, false).getLogs();
+ } catch (IOException e) {
+ LOG.error("Exception in getting local edit log manifest", e);
+ return;
+ }
+
+ GetEditLogManifestResponseProto editLogManifest;
+ try {
+ editLogManifest = jnProxy.getEditLogManifest(null,
+ GetEditLogManifestRequestProto.newBuilder().setJid(jidProto)
+ .setSinceTxId(0)
+ .setInProgressOk(false).build());
+ } catch (ServiceException e) {
+ LOG.error("Could not sync with Journal at " +
+ otherJNProxies.get(journalNodeIndexForSync), e);
+ return;
+ }
+
+ getMissingLogSegments(thisJournalEditLogs, editLogManifest,
+ otherJNProxies.get(index));
+ }
+
+ private List<InetSocketAddress> getOtherJournalNodeAddrs() {
+ URI uri = null;
+ try {
+ String uriStr = conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+ if (uriStr == null || uriStr.isEmpty()) {
+ LOG.warn("Could not construct Shared Edits Uri");
+ return null;
+ }
+ uri = new URI(uriStr);
+ return Util.getLoggerAddresses(uri,
+ Sets.newHashSet(jn.getBoundIpcAddress()));
+ } catch (URISyntaxException e) {
+ LOG.error("The conf property " + DFSConfigKeys
+ .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " not set properly.");
+ } catch (IOException e) {
+ LOG.error("Could not parse JournalNode addresses: " + uri);
+ }
+ return null;
+ }
+
+ private JournalIdProto convertJournalId(String journalId) {
+ return QJournalProtocolProtos.JournalIdProto.newBuilder()
+ .setIdentifier(journalId)
+ .build();
+ }
+
+ private void getMissingLogSegments(List<RemoteEditLog> thisJournalEditLogs,
+ GetEditLogManifestResponseProto response,
+ JournalNodeProxy remoteJNproxy) {
+
+ List<RemoteEditLog> otherJournalEditLogs = PBHelper.convert(
+ response.getManifest()).getLogs();
+ if (otherJournalEditLogs == null || otherJournalEditLogs.isEmpty()) {
+ LOG.warn("Journal at " + remoteJNproxy.jnAddr + " has no edit logs");
+ return;
+ }
+ List<RemoteEditLog> missingLogs = getMissingLogList(thisJournalEditLogs,
+ otherJournalEditLogs);
+
+ if (!missingLogs.isEmpty()) {
+ NamespaceInfo nsInfo = jnStorage.getNamespaceInfo();
+
+ for (RemoteEditLog missingLog : missingLogs) {
+ URL url = null;
+ boolean success = false;
+ try {
+ if (remoteJNproxy.httpServerUrl == null) {
+ if (response.hasFromURL()) {
+ URI uri = URI.create(response.getFromURL());
+ remoteJNproxy.httpServerUrl = getHttpServerURI(uri.getScheme(),
+ uri.getHost(), uri.getPort());
+ } else {
+ remoteJNproxy.httpServerUrl = getHttpServerURI("http",
+ remoteJNproxy.jnAddr.getHostName(), response.getHttpPort());
+ }
+ }
+
+ String urlPath = GetJournalEditServlet.buildPath(jid, missingLog
+ .getStartTxId(), nsInfo);
+ url = new URL(remoteJNproxy.httpServerUrl, urlPath);
+ success = downloadMissingLogSegment(url, missingLog);
+ } catch (MalformedURLException e) {
+ LOG.error("MalformedURL when download missing log segment", e);
+ } catch (Exception e) {
+ LOG.error("Exception in downloading missing log segment from url " +
+ url, e);
+ }
+ if (!success) {
+ LOG.error("Aborting current sync attempt.");
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the logs present in otherJournalEditLogs and missing from
+ * thisJournalEditLogs.
+ */
+ private List<RemoteEditLog> getMissingLogList(
+ List<RemoteEditLog> thisJournalEditLogs,
+ List<RemoteEditLog> otherJournalEditLogs) {
+ if (thisJournalEditLogs.isEmpty()) {
+ return otherJournalEditLogs;
+ }
+
+ List<RemoteEditLog> missingEditLogs = Lists.newArrayList();
+
+ int thisJnIndex = 0, otherJnIndex = 0;
+ int thisJnNumLogs = thisJournalEditLogs.size();
+ int otherJnNumLogs = otherJournalEditLogs.size();
+
+ while (thisJnIndex < thisJnNumLogs && otherJnIndex < otherJnNumLogs) {
+ long localJNstartTxId = thisJournalEditLogs.get(thisJnIndex)
+ .getStartTxId();
+ long remoteJNstartTxId = otherJournalEditLogs.get(otherJnIndex)
+ .getStartTxId();
+
+ if (localJNstartTxId == remoteJNstartTxId) {
+ thisJnIndex++;
+ otherJnIndex++;
+ } else if (localJNstartTxId > remoteJNstartTxId) {
+ missingEditLogs.add(otherJournalEditLogs.get(otherJnIndex));
+ otherJnIndex++;
+ } else {
+ thisJnIndex++;
+ }
+ }
+
+ if (otherJnIndex < otherJnNumLogs) {
+ for (; otherJnIndex < otherJnNumLogs; otherJnIndex++) {
+ missingEditLogs.add(otherJournalEditLogs.get(otherJnIndex));
+ }
+ }
+
+ return missingEditLogs;
+ }
+
+ private URL getHttpServerURI(String scheme, String hostname, int port)
+ throws MalformedURLException {
+ return new URL(scheme, hostname, port, "");
+ }
+
+ /**
+ * Transfer an edit log from one journal node to another for sync-up.
+ */
+ private boolean downloadMissingLogSegment(URL url, RemoteEditLog log) throws
+ IOException {
+ LOG.info("Downloading missing Edit Log from " + url + " to " + jnStorage
+ .getRoot());
+
+ assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : "bad log: " + log;
+ File finalEditsFile = jnStorage.getFinalizedEditsFile(log.getStartTxId(),
+ log.getEndTxId());
+
+ if (finalEditsFile.exists() && FileUtil.canRead(finalEditsFile)) {
+ LOG.info("Skipping download of remote edit log " + log + " since it's" +
+ " already stored locally at " + finalEditsFile);
+ return true;
+ }
+
+ final long milliTime = Time.monotonicNow();
+ File tmpEditsFile = jnStorage.getTemporaryEditsFile(log.getStartTxId(), log
+ .getEndTxId(), milliTime);
+ try {
+ Util.doGetUrl(url, ImmutableList.of(tmpEditsFile), jnStorage, false,
+ logSegmentTransferTimeout, throttler);
+ } catch (IOException e) {
+ LOG.error("Download of Edit Log file for Syncing failed. Deleting temp " +
+ "file: " + tmpEditsFile);
+ if (!tmpEditsFile.delete()) {
+ LOG.warn("Deleting " + tmpEditsFile + " has failed");
+ }
+ return false;
+ }
+ LOG.info("Downloaded file " + tmpEditsFile.getName() + " of size " +
+ tmpEditsFile.length() + " bytes.");
+
+ LOG.debug("Renaming " + tmpEditsFile.getName() + " to "
+ + finalEditsFile.getName());
+ boolean renameSuccess = journal.renameTmpSegment(tmpEditsFile,
+ finalEditsFile, log.getEndTxId());
+ if (!renameSuccess) {
+ //If rename is not successful, delete the tmpFile
+ LOG.debug("Renaming unsuccessful. Deleting temporary file: "
+ + tmpEditsFile);
+ if (!tmpEditsFile.delete()) {
+ LOG.warn("Deleting " + tmpEditsFile + " has failed");
+ }
+ return false;
+ }
+ return true;
+ }
+
+ private static DataTransferThrottler getThrottler(Configuration conf) {
+ long transferBandwidth =
+ conf.getLong(DFSConfigKeys.DFS_EDIT_LOG_TRANSFER_RATE_KEY,
+ DFSConfigKeys.DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT);
+ DataTransferThrottler throttler = null;
+ if (transferBandwidth > 0) {
+ throttler = new DataTransferThrottler(transferBandwidth);
+ }
+ return throttler;
+ }
+
+ private class JournalNodeProxy {
+ private final InetSocketAddress jnAddr;
+ private final QJournalProtocolPB jnProxy;
+ private URL httpServerUrl;
+
+ JournalNodeProxy(InetSocketAddress jnAddr) throws IOException {
+ this.jnAddr = jnAddr;
+ this.jnProxy = RPC.getProxy(QJournalProtocolPB.class,
+ RPC.getProtocolVersion(QJournalProtocolPB.class), jnAddr, conf);
+ }
+
+ @Override
+ public String toString() {
+ return jnAddr.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 1af7877..4493772 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIOException;
import org.apache.hadoop.util.ToolRunner;
@@ -1010,6 +1011,14 @@ public abstract class Storage extends StorageInfo {
return false;
}
+ public NamespaceInfo getNamespaceInfo() {
+ return new NamespaceInfo(
+ getNamespaceID(),
+ getClusterID(),
+ null,
+ getCTime());
+ }
+
/**
* Return true if the layout of the given storage directory is from a version
* of Hadoop prior to the introduction of the "current" and "previous"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index f08c3fa..9c67f0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -22,9 +22,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.net.UnknownHostException;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
@@ -32,18 +34,23 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.server.namenode.ImageServlet;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
@@ -143,7 +150,8 @@ public final class Util {
* storage.
*/
public static MD5Hash doGetUrl(URL url, List<File> localPaths,
- Storage dstStorage, boolean getChecksum, int timeout) throws IOException {
+ Storage dstStorage, boolean getChecksum, int timeout,
+ DataTransferThrottler throttler) throws IOException {
HttpURLConnection connection;
try {
connection = (HttpURLConnection)
@@ -176,7 +184,7 @@ public final class Util {
return receiveFile(url.toExternalForm(), localPaths, dstStorage,
getChecksum, advertisedSize, advertisedDigest, fsImageName, stream,
- null);
+ throttler);
}
/**
@@ -268,7 +276,7 @@ public final class Util {
long xferKb = received / 1024;
xferCombined += xferSec;
xferStats.append(
- String.format(" The fsimage download took %.2fs at %.2f KB/s.",
+ String.format(" The file download took %.2fs at %.2f KB/s.",
xferSec, xferKb / xferSec));
} finally {
stream.close();
@@ -301,7 +309,7 @@ public final class Util {
advertisedSize);
}
}
- xferStats.insert(0, String.format("Combined time for fsimage download and" +
+ xferStats.insert(0, String.format("Combined time for file download and" +
" fsync to all disks took %.2fs.", xferCombined));
LOG.info(xferStats.toString());
@@ -350,4 +358,34 @@ public final class Util {
String header = connection.getHeaderField(MD5_HEADER);
return (header != null) ? new MD5Hash(header) : null;
}
+
+ public static List<InetSocketAddress> getAddressesList(URI uri)
+ throws IOException{
+ String authority = uri.getAuthority();
+ Preconditions.checkArgument(authority != null && !authority.isEmpty(),
+ "URI has no authority: " + uri);
+
+ String[] parts = StringUtils.split(authority, ';');
+ for (int i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].trim();
+ }
+
+ List<InetSocketAddress> addrs = Lists.newArrayList();
+ for (String addr : parts) {
+ InetSocketAddress isa = NetUtils.createSocketAddr(
+ addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT);
+ if (isa.isUnresolved()) {
+ throw new UnknownHostException(addr);
+ }
+ addrs.add(isa);
+ }
+ return addrs;
+ }
+
+ public static List<InetSocketAddress> getLoggerAddresses(URI uri,
+ Set<InetSocketAddress> addrsToExclude) throws IOException {
+ List<InetSocketAddress> addrsList = getAddressesList(uri);
+ addrsList.removeAll(addrsToExclude);
+ return addrsList;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index c79ba4a..63d1a28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -763,13 +763,13 @@ public class NNStorage extends Storage implements Closeable,
return new File(sd.getCurrentDir(), getInProgressEditsFileName(startTxId));
}
- static File getFinalizedEditsFile(StorageDirectory sd,
+ public static File getFinalizedEditsFile(StorageDirectory sd,
long startTxId, long endTxId) {
return new File(sd.getCurrentDir(),
getFinalizedEditsFileName(startTxId, endTxId));
}
- static File getTemporaryEditsFile(StorageDirectory sd,
+ public static File getTemporaryEditsFile(StorageDirectory sd,
long startTxId, long endTxId, long timestamp) {
return new File(sd.getCurrentDir(),
getTemporaryEditsFileName(startTxId, endTxId, timestamp));
@@ -1106,6 +1106,7 @@ public class NNStorage extends Storage implements Closeable,
return inspector;
}
+ @Override
public NamespaceInfo getNamespaceInfo() {
return new NamespaceInfo(
getNamespaceID(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 5821353..7316414 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -401,7 +401,8 @@ public class TransferFsImage {
public static MD5Hash doGetUrl(URL url, List<File> localPaths,
Storage dstStorage, boolean getChecksum) throws IOException {
- return Util.doGetUrl(url, localPaths, dstStorage, getChecksum, timeout);
+ return Util.doGetUrl(url, localPaths, dstStorage, getChecksum, timeout,
+ null);
}
private static MD5Hash parseMD5Header(HttpServletRequest request) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 03f1a08..652b216 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1279,6 +1279,26 @@
</property>
<property>
+ <name>dfs.edit.log.transfer.timeout</name>
+ <value>30000</value>
+ <description>
+ Socket timeout for edit log transfer in milliseconds. This timeout
+ should be configured such that normal edit log transfer for journal
+ node syncing can complete successfully.
+ </description>
+</property>
+
+<property>
+ <name>dfs.edit.log.transfer.bandwidthPerSec</name>
+ <value>0</value>
+ <description>
+ Maximum bandwidth used for transferring edit log to between journal nodes
+ for syncing, in bytes per second.
+ A default value of 0 indicates that throttling is disabled.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.support.allow.format</name>
<value>true</value>
<description>Does HDFS namenode allow itself to be formatted?
@@ -3785,6 +3805,27 @@
</property>
<property>
+ <name>dfs.journalnode.enable.sync</name>
+ <value>true</value>
+ <description>
+ If true, the journal nodes wil sync with each other. The journal nodes
+ will periodically gossip with other journal nodes to compare edit log
+ manifests and if they detect any missing log segment, they will download
+ it from the other journal nodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.journalnode.sync.interval</name>
+ <value>120000</value>
+ <description>
+ Time interval, in milliseconds, between two Journal Node syncs.
+ This configuration takes effect only if the journalnode sync is enabled
+ by setting the configuration parameter dfs.journalnode.enable.sync to true.
+ </description>
+</property>
+
+<property>
<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
<value></value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
index 7b974c3..2314e22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
@@ -255,4 +255,12 @@ public class MiniJournalCluster {
}
}
}
+
+ public void setNamenodeSharedEditsConf(String jid) {
+ URI quorumJournalURI = getQuorumJournalURI(jid);
+ for (int i = 0; i < nodes.length; i++) {
+ nodes[i].node.getConf().set(DFSConfigKeys
+ .DFS_NAMENODE_SHARED_EDITS_DIR_KEY, quorumJournalURI.toString());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
index 0764f12..c163894 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
@@ -101,6 +101,7 @@ public class MiniQJMHACluster {
journalCluster = new MiniJournalCluster.Builder(conf).format(true)
.build();
journalCluster.waitActive();
+ journalCluster.setNamenodeSharedEditsConf(NAMESERVICE);
URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE);
// start cluster with specified NameNodes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d4bcfe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
new file mode 100644
index 0000000..5375b02
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestJournalNodeSync.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.qjournal;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
+import static org.apache.hadoop.hdfs.server.namenode.FileJournalManager
+ .getLogFile;
+
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Unit test for Journal Node formatting upon re-installation and syncing.
+ */
+public class TestJournalNodeSync {
+ private MiniQJMHACluster qjmhaCluster;
+ private MiniDFSCluster dfsCluster;
+ private MiniJournalCluster jCluster;
+ private FileSystem fs;
+ private FSNamesystem namesystem;
+ private int editsPerformed = 0;
+ private final String jid = "ns1";
+
+ @Before
+ public void setUpMiniCluster() throws IOException {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_JOURNALNODE_SYNC_INTERVAL_KEY, 1000L);
+ qjmhaCluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(2)
+ .build();
+ dfsCluster = qjmhaCluster.getDfsCluster();
+ jCluster = qjmhaCluster.getJournalCluster();
+
+ dfsCluster.transitionToActive(0);
+ fs = dfsCluster.getFileSystem(0);
+ namesystem = dfsCluster.getNamesystem(0);
+ }
+
+ @After
+ public void shutDownMiniCluster() throws IOException {
+ if (qjmhaCluster != null) {
+ qjmhaCluster.shutdown();
+ }
+ }
+
+ @Test(timeout=30000)
+ public void testJournalNodeSync() throws Exception {
+ File firstJournalDir = jCluster.getJournalDir(0, jid);
+ File firstJournalCurrentDir = new StorageDirectory(firstJournalDir)
+ .getCurrentDir();
+
+ // Generate some edit logs and delete one.
+ long firstTxId = generateEditLog();
+ generateEditLog();
+
+ File missingLog = deleteEditLog(firstJournalCurrentDir, firstTxId);
+
+ GenericTestUtils.waitFor(editLogExists(Lists.newArrayList(missingLog)),
+ 500, 10000);
+ }
+
+ @Test(timeout=30000)
+ public void testSyncForMultipleMissingLogs() throws Exception {
+ File firstJournalDir = jCluster.getJournalDir(0, jid);
+ File firstJournalCurrentDir = new StorageDirectory(firstJournalDir)
+ .getCurrentDir();
+
+ // Generate some edit logs and delete two.
+ long firstTxId = generateEditLog();
+ long nextTxId = generateEditLog();
+
+ List<File> missingLogs = Lists.newArrayList();
+ missingLogs.add(deleteEditLog(firstJournalCurrentDir, firstTxId));
+ missingLogs.add(deleteEditLog(firstJournalCurrentDir, nextTxId));
+
+ GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 10000);
+ }
+
+ @Test(timeout=30000)
+ public void testSyncForDiscontinuousMissingLogs() throws Exception {
+ File firstJournalDir = jCluster.getJournalDir(0, jid);
+ File firstJournalCurrentDir = new StorageDirectory(firstJournalDir)
+ .getCurrentDir();
+
+ // Generate some edit logs and delete two discontinuous logs.
+ long firstTxId = generateEditLog();
+ generateEditLog();
+ long nextTxId = generateEditLog();
+
+ List<File> missingLogs = Lists.newArrayList();
+ missingLogs.add(deleteEditLog(firstJournalCurrentDir, firstTxId));
+ missingLogs.add(deleteEditLog(firstJournalCurrentDir, nextTxId));
+
+ GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 10000);
+ }
+
+ @Test(timeout=30000)
+ public void testMultipleJournalsMissingLogs() throws Exception {
+ File firstJournalDir = jCluster.getJournalDir(0, jid);
+ File firstJournalCurrentDir = new StorageDirectory(firstJournalDir)
+ .getCurrentDir();
+
+ File secondJournalDir = jCluster.getJournalDir(1, jid);
+ StorageDirectory sd = new StorageDirectory(secondJournalDir);
+ File secondJournalCurrentDir = sd.getCurrentDir();
+
+ // Generate some edit logs and delete one log from two journals.
+ long firstTxId = generateEditLog();
+ generateEditLog();
+
+ List<File> missingLogs = Lists.newArrayList();
+ missingLogs.add(deleteEditLog(firstJournalCurrentDir, firstTxId));
+ missingLogs.add(deleteEditLog(secondJournalCurrentDir, firstTxId));
+
+ GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 10000);
+ }
+
+ @Test(timeout=60000)
+ public void testMultipleJournalsMultipleMissingLogs() throws Exception {
+ File firstJournalDir = jCluster.getJournalDir(0, jid);
+ File firstJournalCurrentDir = new StorageDirectory(firstJournalDir)
+ .getCurrentDir();
+
+ File secondJournalDir = jCluster.getJournalDir(1, jid);
+ File secondJournalCurrentDir = new StorageDirectory(secondJournalDir)
+ .getCurrentDir();
+
+ File thirdJournalDir = jCluster.getJournalDir(2, jid);
+ File thirdJournalCurrentDir = new StorageDirectory(thirdJournalDir)
+ .getCurrentDir();
+
+ // Generate some edit logs and delete multiple logs in multiple journals.
+ long firstTxId = generateEditLog();
+ long secondTxId = generateEditLog();
+ long thirdTxId = generateEditLog();
+
+ List<File> missingLogs = Lists.newArrayList();
+ missingLogs.add(deleteEditLog(firstJournalCurrentDir, firstTxId));
+ missingLogs.add(deleteEditLog(secondJournalCurrentDir, firstTxId));
+ missingLogs.add(deleteEditLog(secondJournalCurrentDir, secondTxId));
+ missingLogs.add(deleteEditLog(thirdJournalCurrentDir, thirdTxId));
+
+ GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 30000);
+ }
+
+ // Test JournalNode Sync by randomly deleting edit logs from one or two of
+ // the journals.
+ @Test(timeout=60000)
+ public void testRandomJournalMissingLogs() throws Exception {
+ Random randomJournal = new Random();
+
+ List<File> journalCurrentDirs = Lists.newArrayList();
+
+ for (int i = 0; i < 3; i++) {
+ journalCurrentDirs.add(new StorageDirectory(jCluster.getJournalDir(i,
+ jid)).getCurrentDir());
+ }
+
+ int count = 0;
+ long lastStartTxId;
+ int journalIndex;
+ List<File> missingLogs = Lists.newArrayList();
+ while (count < 5) {
+ lastStartTxId = generateEditLog();
+
+ // Delete the last edit log segment from randomly selected journal node
+ journalIndex = randomJournal.nextInt(3);
+ missingLogs.add(deleteEditLog(journalCurrentDirs.get(journalIndex),
+ lastStartTxId));
+
+ // Delete the last edit log segment from two journals for some logs
+ if (count % 2 == 0) {
+ journalIndex = (journalIndex + 1) % 3;
+ missingLogs.add(deleteEditLog(journalCurrentDirs.get(journalIndex),
+ lastStartTxId));
+ }
+
+ count++;
+ }
+
+ GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 30000);
+ }
+
+ private File deleteEditLog(File currentDir, long startTxId)
+ throws IOException {
+ EditLogFile logFile = getLogFile(currentDir, startTxId);
+ while (logFile.isInProgress()) {
+ dfsCluster.getNameNode(0).getRpcServer().rollEditLog();
+ logFile = getLogFile(currentDir, startTxId);
+ }
+ File deleteFile = logFile.getFile();
+ Assert.assertTrue("Couldn't delete edit log file", deleteFile.delete());
+
+ return deleteFile;
+ }
+
+ /**
+ * Do a mutative metadata operation on the file system.
+ *
+ * @return true if the operation was successful, false otherwise.
+ */
+ private boolean doAnEdit() throws IOException {
+ return fs.mkdirs(new Path("/tmp", Integer.toString(editsPerformed++)));
+ }
+
+ /**
+ * Does an edit and rolls the Edit Log.
+ *
+ * @return the startTxId of next segment after rolling edits.
+ */
+ private long generateEditLog() throws IOException {
+ long startTxId = namesystem.getFSImage().getEditLog().getLastWrittenTxId();
+ Assert.assertTrue("Failed to do an edit", doAnEdit());
+ dfsCluster.getNameNode(0).getRpcServer().rollEditLog();
+ return startTxId;
+ }
+
+ private Supplier<Boolean> editLogExists(List<File> editLogs) {
+ Supplier<Boolean> supplier = new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ for (File editLog : editLogs) {
+ if (!editLog.exists()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ };
+ return supplier;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-5519. Add SubClusterId in
AddApplicationHomeSubClusterResponse for Router Failover. (Ellen Hui via
Subru)
Posted by su...@apache.org.
YARN-5519. Add SubClusterId in AddApplicationHomeSubClusterResponse for Router Failover. (Ellen Hui via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd9ed274
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd9ed274
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd9ed274
Branch: refs/heads/YARN-2915
Commit: dd9ed274e7f2b80b10868437721292f36bed67f1
Parents: d9abcba
Author: Subru Krishnan <su...@apache.org>
Authored: Mon Aug 15 14:47:02 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
...ederationApplicationHomeSubClusterStore.java | 21 +++---
.../store/impl/MemoryFederationStateStore.java | 22 +++---
.../AddApplicationHomeSubClusterResponse.java | 29 ++++++--
...ApplicationHomeSubClusterResponsePBImpl.java | 39 +++++++++++
.../proto/yarn_server_federation_protos.proto | 1 +
.../impl/FederationStateStoreBaseTest.java | 71 +++++++++-----------
6 files changed, 120 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9ed274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
index 22bb88a..ace2457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -51,15 +51,20 @@ public interface FederationApplicationHomeSubClusterStore {
/**
* Register the home {@code SubClusterId} of the newly submitted
* {@code ApplicationId}. Currently response is empty if the operation was
- * successful, if not an exception reporting reason for a failure.
+ * successful, if not an exception reporting reason for a failure. If a
+ * mapping for the application already existed, the {@code SubClusterId} in
+ * this response will return the existing mapping which might be different
+ * from that in the {@code AddApplicationHomeSubClusterRequest}.
*
* @param request the request to register a new application with its home
* sub-cluster
- * @return empty on successful registration of the application in the
- * StateStore, if not an exception reporting reason for a failure
+ * @return upon successful registration of the application in the StateStore,
+ * {@code AddApplicationHomeSubClusterRequest} containing the home
+ * sub-cluster of the application. Otherwise, an exception reporting
+ * reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -73,7 +78,7 @@ public interface FederationApplicationHomeSubClusterStore {
* not an exception reporting reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -85,7 +90,7 @@ public interface FederationApplicationHomeSubClusterStore {
* subcluster
* @throws YarnException if the request is invalid/fails
*/
- GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -96,7 +101,7 @@ public interface FederationApplicationHomeSubClusterStore {
* @return the mapping of all submitted application to it's home sub-cluster
* @throws YarnException if the request is invalid/fails
*/
- GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException;
/**
@@ -110,7 +115,7 @@ public interface FederationApplicationHomeSubClusterStore {
* not an exception reporting reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
DeleteApplicationHomeSubClusterRequest request) throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9ed274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index a540dff..8144435 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -89,7 +89,6 @@ public class MemoryFederationStateStore implements FederationStateStore {
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
- subClusterInfo.setLastStartTime(clock.getTime());
membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
return SubClusterRegisterResponse.newInstance();
}
@@ -156,21 +155,22 @@ public class MemoryFederationStateStore implements FederationStateStore {
// FederationApplicationHomeSubClusterStore methods
@Override
- public AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
- if (applications.containsKey(appId)) {
- throw new YarnException("Application " + appId + " already exists");
+
+ if (!applications.containsKey(appId)) {
+ applications.put(appId,
+ request.getApplicationHomeSubCluster().getHomeSubCluster());
}
- applications.put(appId,
- request.getApplicationHomeSubCluster().getHomeSubCluster());
- return AddApplicationHomeSubClusterResponse.newInstance();
+ return AddApplicationHomeSubClusterResponse
+ .newInstance(applications.get(appId));
}
@Override
- public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId =
request.getApplicationHomeSubCluster().getApplicationId();
@@ -184,7 +184,7 @@ public class MemoryFederationStateStore implements FederationStateStore {
}
@Override
- public GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
@@ -196,7 +196,7 @@ public class MemoryFederationStateStore implements FederationStateStore {
}
@Override
- public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
List<ApplicationHomeSubCluster> result =
new ArrayList<ApplicationHomeSubCluster>();
@@ -210,7 +210,7 @@ public class MemoryFederationStateStore implements FederationStateStore {
}
@Override
- public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
DeleteApplicationHomeSubClusterRequest request) throws YarnException {
ApplicationId appId = request.getApplicationId();
if (!applications.containsKey(appId)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9ed274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
index 2145dd1..913f8e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
@@ -24,10 +24,13 @@ import org.apache.hadoop.yarn.util.Records;
/**
* AddApplicationHomeSubClusterResponse contains the answer from the
* {@code FederationApplicationHomeSubClusterStore} to a request to insert a
- * newly generated applicationId and its owner. Currently response is empty if
- * the operation was successful, if not an exception reporting reason for a
- * failure.
+ * newly generated applicationId and its owner.
*
+ * The response contains application's home sub-cluster as it is stored in the
+ * {@code FederationApplicationHomeSubClusterStore}. If a mapping for the
+ * application already existed, the {@code SubClusterId} in this response will
+ * return the existing mapping which might be different from that in the
+ * {@code AddApplicationHomeSubClusterRequest}.
*/
@Private
@Unstable
@@ -35,10 +38,28 @@ public abstract class AddApplicationHomeSubClusterResponse {
@Private
@Unstable
- public static AddApplicationHomeSubClusterResponse newInstance() {
+ public static AddApplicationHomeSubClusterResponse newInstance(
+ SubClusterId homeSubCluster) {
AddApplicationHomeSubClusterResponse response =
Records.newRecord(AddApplicationHomeSubClusterResponse.class);
+ response.setHomeSubCluster(homeSubCluster);
return response;
}
+ /**
+ * Set the home sub-cluster that this application has been assigned to.
+ *
+ * @param homeSubCluster the {@link SubClusterId} of this application's home
+ * sub-cluster
+ */
+ public abstract void setHomeSubCluster(SubClusterId homeSubCluster);
+
+ /**
+ * Get the home sub-cluster that this application has been assigned to. This
+ * may not match the {@link SubClusterId} in the corresponding response, if
+ * the mapping for the request's application already existed.
+ *
+ * @return the {@link SubClusterId} of this application's home sub-cluster
+ */
+ public abstract SubClusterId getHomeSubCluster();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9ed274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
index 1415981..b30c41c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import com.google.protobuf.TextFormat;
@@ -48,6 +51,13 @@ public class AddApplicationHomeSubClusterResponsePBImpl
viaProto = true;
}
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = AddApplicationHomeSubClusterResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
public AddApplicationHomeSubClusterResponseProto getProto() {
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -55,6 +65,27 @@ public class AddApplicationHomeSubClusterResponsePBImpl
}
@Override
+ public void setHomeSubCluster(SubClusterId homeSubCluster) {
+ maybeInitBuilder();
+ if (homeSubCluster == null) {
+ builder.clearHomeSubCluster();
+ return;
+ }
+ builder.setHomeSubCluster(convertToProtoFormat(homeSubCluster));
+ }
+
+ @Override
+ public SubClusterId getHomeSubCluster() {
+ AddApplicationHomeSubClusterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+
+ if (!p.hasHomeSubCluster()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getHomeSubCluster());
+ }
+
+ @Override
public int hashCode() {
return getProto().hashCode();
}
@@ -75,4 +106,12 @@ public class AddApplicationHomeSubClusterResponsePBImpl
return TextFormat.shortDebugString(getProto());
}
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+ return new SubClusterIdPBImpl(sc);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+ return ((SubClusterIdPBImpl) sc).getProto();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9ed274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 11f786f..cedf482 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -102,6 +102,7 @@ message AddApplicationHomeSubClusterRequestProto {
}
message AddApplicationHomeSubClusterResponseProto {
+ optional SubClusterIdProto home_sub_cluster = 1;
}
message UpdateApplicationHomeSubClusterRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd9ed274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index 165dd78..414696b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -94,7 +94,7 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testDeregisterSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- registerSubCluster(subClusterId);
+ registerSubCluster(createSubClusterInfo(subClusterId));
SubClusterDeregisterRequest deregisterRequest = SubClusterDeregisterRequest
.newInstance(subClusterId, SubClusterState.SC_UNREGISTERED);
@@ -124,7 +124,7 @@ public abstract class FederationStateStoreBaseTest {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
- registerSubCluster(subClusterId);
+ registerSubCluster(subClusterInfo);
GetSubClusterInfoRequest request =
GetSubClusterInfoRequest.newInstance(subClusterId);
@@ -184,10 +184,10 @@ public abstract class FederationStateStoreBaseTest {
@Test
public void testSubClusterHeartbeat() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
- registerSubCluster(subClusterId);
+ registerSubCluster(createSubClusterInfo(subClusterId));
SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
- .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "capability");
stateStore.subClusterHeartbeat(heartbeatRequest);
Assert.assertEquals(SubClusterState.SC_RUNNING,
@@ -199,7 +199,7 @@ public abstract class FederationStateStoreBaseTest {
public void testSubClusterHeartbeatUnknownSubCluster() throws Exception {
SubClusterId subClusterId = SubClusterId.newInstance("SC");
SubClusterHeartbeatRequest heartbeatRequest = SubClusterHeartbeatRequest
- .newInstance(subClusterId, SubClusterState.SC_RUNNING, "cabability");
+ .newInstance(subClusterId, SubClusterState.SC_RUNNING, "capability");
try {
stateStore.subClusterHeartbeat(heartbeatRequest);
@@ -213,7 +213,7 @@ public abstract class FederationStateStoreBaseTest {
// Test FederationApplicationHomeSubClusterStore
@Test
- public void testAddApplicationHomeSubClusterMap() throws Exception {
+ public void testAddApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId = SubClusterId.newInstance("SC");
ApplicationHomeSubCluster ahsc =
@@ -222,15 +222,15 @@ public abstract class FederationStateStoreBaseTest {
AddApplicationHomeSubClusterRequest request =
AddApplicationHomeSubClusterRequest.newInstance(ahsc);
AddApplicationHomeSubClusterResponse response =
- stateStore.addApplicationHomeSubClusterMap(request);
+ stateStore.addApplicationHomeSubCluster(request);
- Assert.assertNotNull(response);
+ Assert.assertEquals(subClusterId, response.getHomeSubCluster());
Assert.assertEquals(subClusterId, queryApplicationHomeSC(appId));
}
@Test
- public void testAddApplicationHomeSubClusterMapAppAlreadyExists()
+ public void testAddApplicationHomeSubClusterAppAlreadyExists()
throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
@@ -240,21 +240,17 @@ public abstract class FederationStateStoreBaseTest {
ApplicationHomeSubCluster ahsc2 =
ApplicationHomeSubCluster.newInstance(appId, subClusterId2);
- try {
- stateStore.addApplicationHomeSubClusterMap(
- AddApplicationHomeSubClusterRequest.newInstance(ahsc2));
- Assert.fail();
- } catch (YarnException e) {
- Assert.assertTrue(e.getMessage()
- .startsWith("Application " + appId.toString() + " already exists"));
- }
+ AddApplicationHomeSubClusterResponse response =
+ stateStore.addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest.newInstance(ahsc2));
+ Assert.assertEquals(subClusterId1, response.getHomeSubCluster());
Assert.assertEquals(subClusterId1, queryApplicationHomeSC(appId));
}
@Test
- public void testDeleteApplicationHomeSubClusterMap() throws Exception {
+ public void testDeleteApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId = SubClusterId.newInstance("SC");
addApplicationHomeSC(appId, subClusterId);
@@ -263,7 +259,7 @@ public abstract class FederationStateStoreBaseTest {
DeleteApplicationHomeSubClusterRequest.newInstance(appId);
DeleteApplicationHomeSubClusterResponse response =
- stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+ stateStore.deleteApplicationHomeSubCluster(delRequest);
Assert.assertNotNull(response);
try {
@@ -277,14 +273,13 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testDeleteApplicationHomeSubClusterMapUnknownApp()
- throws Exception {
+ public void testDeleteApplicationHomeSubClusterUnknownApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
DeleteApplicationHomeSubClusterRequest delRequest =
DeleteApplicationHomeSubClusterRequest.newInstance(appId);
try {
- stateStore.deleteApplicationHomeSubClusterMap(delRequest);
+ stateStore.deleteApplicationHomeSubCluster(delRequest);
Assert.fail();
} catch (YarnException e) {
Assert.assertTrue(e.getMessage()
@@ -293,7 +288,7 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testGetApplicationHomeSubClusterMap() throws Exception {
+ public void testGetApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId = SubClusterId.newInstance("SC");
addApplicationHomeSC(appId, subClusterId);
@@ -302,7 +297,7 @@ public abstract class FederationStateStoreBaseTest {
GetApplicationHomeSubClusterRequest.newInstance(appId);
GetApplicationHomeSubClusterResponse result =
- stateStore.getApplicationHomeSubClusterMap(getRequest);
+ stateStore.getApplicationHomeSubCluster(getRequest);
Assert.assertEquals(appId,
result.getApplicationHomeSubCluster().getApplicationId());
@@ -311,13 +306,13 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testGetApplicationHomeSubClusterMapUnknownApp() throws Exception {
+ public void testGetApplicationHomeSubClusterUnknownApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
GetApplicationHomeSubClusterRequest request =
GetApplicationHomeSubClusterRequest.newInstance(appId);
try {
- stateStore.getApplicationHomeSubClusterMap(request);
+ stateStore.getApplicationHomeSubCluster(request);
Assert.fail();
} catch (YarnException e) {
Assert.assertTrue(e.getMessage()
@@ -326,7 +321,7 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testGetApplicationsHomeSubClusterMap() throws Exception {
+ public void testGetApplicationsHomeSubCluster() throws Exception {
ApplicationId appId1 = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
ApplicationHomeSubCluster ahsc1 =
@@ -344,7 +339,7 @@ public abstract class FederationStateStoreBaseTest {
GetApplicationsHomeSubClusterRequest.newInstance();
GetApplicationsHomeSubClusterResponse result =
- stateStore.getApplicationsHomeSubClusterMap(getRequest);
+ stateStore.getApplicationsHomeSubCluster(getRequest);
Assert.assertEquals(2, result.getAppsHomeSubClusters().size());
Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc1));
@@ -352,7 +347,7 @@ public abstract class FederationStateStoreBaseTest {
}
@Test
- public void testUpdateApplicationHomeSubClusterMap() throws Exception {
+ public void testUpdateApplicationHomeSubCluster() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
addApplicationHomeSC(appId, subClusterId1);
@@ -365,16 +360,14 @@ public abstract class FederationStateStoreBaseTest {
UpdateApplicationHomeSubClusterRequest.newInstance(ahscUpdate);
UpdateApplicationHomeSubClusterResponse response =
- stateStore.updateApplicationHomeSubClusterMap(updateRequest);
+ stateStore.updateApplicationHomeSubCluster(updateRequest);
Assert.assertNotNull(response);
-
Assert.assertEquals(subClusterId2, queryApplicationHomeSC(appId));
}
@Test
- public void testUpdateApplicationHomeSubClusterMapUnknownApp()
- throws Exception {
+ public void testUpdateApplicationHomeSubClusterUnknownApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(1, 1);
SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
ApplicationHomeSubCluster ahsc =
@@ -384,7 +377,7 @@ public abstract class FederationStateStoreBaseTest {
UpdateApplicationHomeSubClusterRequest.newInstance(ahsc);
try {
- stateStore.updateApplicationHomeSubClusterMap((updateRequest));
+ stateStore.updateApplicationHomeSubCluster((updateRequest));
Assert.fail();
} catch (YarnException e) {
Assert.assertTrue(e.getMessage()
@@ -484,7 +477,7 @@ public abstract class FederationStateStoreBaseTest {
return SubClusterInfo.newInstance(subClusterId, amRMAddress,
clientRMAddress, rmAdminAddress, webAppAddress, SubClusterState.SC_NEW,
- CLOCK.getTime(), "cabability");
+ CLOCK.getTime(), "capability");
}
private SubClusterPolicyConfiguration createSCPolicyConf(String queueName,
@@ -499,7 +492,7 @@ public abstract class FederationStateStoreBaseTest {
ApplicationHomeSubCluster.newInstance(appId, subClusterId);
AddApplicationHomeSubClusterRequest request =
AddApplicationHomeSubClusterRequest.newInstance(ahsc);
- stateStore.addApplicationHomeSubClusterMap(request);
+ stateStore.addApplicationHomeSubCluster(request);
}
private void setPolicyConf(String queue, String policyType)
@@ -510,10 +503,8 @@ public abstract class FederationStateStoreBaseTest {
stateStore.setPolicyConfiguration(request);
}
- private void registerSubCluster(SubClusterId subClusterId)
+ private void registerSubCluster(SubClusterInfo subClusterInfo)
throws YarnException {
-
- SubClusterInfo subClusterInfo = createSubClusterInfo(subClusterId);
stateStore.registerSubCluster(
SubClusterRegisterRequest.newInstance(subClusterInfo));
}
@@ -531,7 +522,7 @@ public abstract class FederationStateStoreBaseTest {
GetApplicationHomeSubClusterRequest.newInstance(appId);
GetApplicationHomeSubClusterResponse response =
- stateStore.getApplicationHomeSubClusterMap(request);
+ stateStore.getApplicationHomeSubCluster(request);
return response.getApplicationHomeSubCluster().getHomeSubCluster();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-5324. Stateless Federation
router policies implementation. (Carlo Curino via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f03e0f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index 8c2115b..f901329 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -19,13 +19,20 @@ package org.apache.hadoop.yarn.server.federation.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
-import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.*;
import java.net.URL;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
@@ -41,6 +48,41 @@ public final class FederationPoliciesTestUtil {
// disabled.
}
+
+ public static void initializePolicyContext(
+ FederationPolicyInitializationContext fpc, ConfigurableFederationPolicy
+ policy, WeightedPolicyInfo policyInfo,
+ Map<SubClusterId, SubClusterInfo> activeSubclusters)
+ throws YarnException {
+ ByteBuffer buf = policyInfo.toByteBuffer();
+ fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+ .newInstance("queue1", policy.getClass().getCanonicalName(), buf));
+ FederationStateStoreFacade facade = FederationStateStoreFacade
+ .getInstance();
+ FederationStateStore fss = mock(FederationStateStore.class);
+
+ if (activeSubclusters == null) {
+ activeSubclusters = new HashMap<SubClusterId, SubClusterInfo>();
+ }
+ GetSubClustersInfoResponse response = GetSubClustersInfoResponse
+ .newInstance(new ArrayList<SubClusterInfo>(activeSubclusters.values()));
+
+ when(fss.getSubClusters(any())).thenReturn(response);
+ facade.reinitialize(fss, new Configuration());
+ fpc.setFederationStateStoreFacade(facade);
+ policy.reinitialize(fpc);
+ }
+
+ public static void initializePolicyContext(
+ ConfigurableFederationPolicy policy,
+ WeightedPolicyInfo policyInfo, Map<SubClusterId,
+ SubClusterInfo> activeSubclusters) throws YarnException {
+ FederationPolicyInitializationContext context =
+ new FederationPolicyInitializationContext(null, initResolver(),
+ initFacade());
+ initializePolicyContext(context, policy, policyInfo, activeSubclusters);
+ }
+
/**
* Initialize a {@link SubClusterResolver}.
*
@@ -66,18 +108,52 @@ public final class FederationPoliciesTestUtil {
* Initialiaze a main-memory {@link FederationStateStoreFacade} used for
* testing, wiht a mock resolver.
*
+ * @param subClusterInfos the list of subclusters to be served on
+ * getSubClusters invocations.
+ *
* @return the facade.
*
* @throws YarnException in case the initialization is not successful.
*/
- public static FederationStateStoreFacade initFacade() throws YarnException {
+
+ public static FederationStateStoreFacade initFacade(
+ List<SubClusterInfo> subClusterInfos, SubClusterPolicyConfiguration
+ policyConfiguration) throws YarnException {
FederationStateStoreFacade goodFacade = FederationStateStoreFacade
.getInstance();
FederationStateStore fss = mock(FederationStateStore.class);
GetSubClustersInfoResponse response = GetSubClustersInfoResponse
- .newInstance(new ArrayList<>());
+ .newInstance(subClusterInfos);
when(fss.getSubClusters(any())).thenReturn(response);
+
+ List<SubClusterPolicyConfiguration> configurations = new ArrayList<>();
+ configurations.add(policyConfiguration);
+
+ GetSubClusterPoliciesConfigurationsResponse policiesResponse =
+ GetSubClusterPoliciesConfigurationsResponse
+ .newInstance(configurations);
+ when(fss.getPoliciesConfigurations(any())).thenReturn(policiesResponse);
+
+ GetSubClusterPolicyConfigurationResponse policyResponse =
+ GetSubClusterPolicyConfigurationResponse
+ .newInstance(policyConfiguration);
+ when(fss.getPolicyConfiguration(any())).thenReturn(policyResponse);
+
goodFacade.reinitialize(fss, new Configuration());
return goodFacade;
}
+
+ /**
+ * Initialiaze a main-memory {@link FederationStateStoreFacade} used for
+ * testing, wiht a mock resolver.
+ *
+ * @return the facade.
+ *
+ * @throws YarnException in case the initialization is not successful.
+ */
+ public static FederationStateStoreFacade initFacade() throws YarnException {
+ return initFacade(new ArrayList<>(), mock(SubClusterPolicyConfiguration
+ .class));
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-5307. Federation Application
State Store internal APIs
Posted by su...@apache.org.
YARN-5307. Federation Application State Store internal APIs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2d3d362
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2d3d362
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2d3d362
Branch: refs/heads/YARN-2915
Commit: b2d3d362ab4545e3da66b402c8e2715415d1354d
Parents: 934c3d5
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Aug 5 11:52:44 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Feb 24 11:31:44 2017 -0800
----------------------------------------------------------------------
...ederationApplicationHomeSubClusterStore.java | 126 ++++++++++++
.../AddApplicationHomeSubClusterRequest.java | 72 +++++++
.../AddApplicationHomeSubClusterResponse.java | 44 +++++
.../records/ApplicationHomeSubCluster.java | 124 ++++++++++++
.../DeleteApplicationHomeSubClusterRequest.java | 65 +++++++
...DeleteApplicationHomeSubClusterResponse.java | 43 +++++
.../GetApplicationHomeSubClusterRequest.java | 64 +++++++
.../GetApplicationHomeSubClusterResponse.java | 73 +++++++
.../GetApplicationsHomeSubClusterRequest.java | 40 ++++
.../GetApplicationsHomeSubClusterResponse.java | 75 ++++++++
.../UpdateApplicationHomeSubClusterRequest.java | 74 ++++++++
...UpdateApplicationHomeSubClusterResponse.java | 43 +++++
...dApplicationHomeSubClusterRequestPBImpl.java | 132 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 78 ++++++++
.../pb/ApplicationHomeSubClusterPBImpl.java | 167 ++++++++++++++++
...eApplicationHomeSubClusterRequestPBImpl.java | 130 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 78 ++++++++
...tApplicationHomeSubClusterRequestPBImpl.java | 135 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 132 +++++++++++++
...ApplicationsHomeSubClusterRequestPBImpl.java | 78 ++++++++
...pplicationsHomeSubClusterResponsePBImpl.java | 190 +++++++++++++++++++
.../pb/GetSubClustersInfoResponsePBImpl.java | 6 +-
...eApplicationHomeSubClusterRequestPBImpl.java | 132 +++++++++++++
...ApplicationHomeSubClusterResponsePBImpl.java | 78 ++++++++
.../proto/yarn_server_federation_protos.proto | 45 ++++-
.../records/TestFederationProtocolRecords.java | 81 ++++++++
26 files changed, 2301 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
new file mode 100644
index 0000000..217ee2e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+
+/**
+ * FederationApplicationHomeSubClusterStore maintains the state of all
+ * <em>Applications</em> that have been submitted to the federated cluster.
+ *
+ * *
+ * <p>
+ * The mapping details contains:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ *
+ */
+@Private
+@Unstable
+public interface FederationApplicationHomeSubClusterStore {
+
+ /**
+ * Get the {@link Version} of the underlying federation application state
+ * store.
+ *
+ * @return the {@link Version} of the underlying federation application state
+ * store
+ */
+ Version getApplicationStateStoreVersion();
+
+ /**
+ * Register the home {@code SubClusterId} of the newly submitted
+ * {@code ApplicationId}. Currently response is empty if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @param request the request to register a new application with its home
+ * sub-cluster
+ * @return empty on successful registration of the application in the
+ * StateStore, if not an exception reporting reason for a failure
+ * @throws YarnException if the request is invalid/fails
+ */
+ AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Update the home {@code SubClusterId} of a previously submitted
+ * {@code ApplicationId}. Currently response is empty if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @param request the request to update the home sub-cluster of an
+ * application.
+ * @return empty on successful update of the application in the StateStore, if
+ * not an exception reporting reason for a failure
+ * @throws YarnException if the request is invalid/fails
+ */
+ UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Get information about the application identified by the input
+ * {@code ApplicationId}.
+ *
+ * @param request contains the application queried
+ * @return {@code ApplicationHomeSubCluster} containing the application's
+ * home subcluster
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Get the {@code ApplicationHomeSubCluster} list representing the mapping
+ * of all submitted applications to it's home sub-cluster.
+ *
+ * @param request empty representing all applications
+ * @return the mapping of all submitted application to it's home sub-cluster
+ * @throws YarnException if the request is invalid/fails
+ */
+ GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException;
+
+ /**
+ * Delete the mapping of home {@code SubClusterId} of a previously submitted
+ * {@code ApplicationId}. Currently response is empty if the operation was
+ * successful, if not an exception reporting reason for a failure.
+ *
+ * @param request the request to delete the home sub-cluster of an
+ * application.
+ * @return empty on successful update of the application in the StateStore, if
+ * not an exception reporting reason for a failure
+ * @throws YarnException if the request is invalid/fails
+ */
+ DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubClusterMap(
+ DeleteApplicationHomeSubClusterRequest request) throws YarnException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..9cb0589
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterRequest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent by the <code>Router</code> to <code>Federation state
+ * store</code> to map the home subcluster of a newly submitted application.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class AddApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static AddApplicationHomeSubClusterRequest newInstance(
+ ApplicationHomeSubCluster applicationHomeSubCluster) {
+ AddApplicationHomeSubClusterRequest mapRequest =
+ Records.newRecord(AddApplicationHomeSubClusterRequest.class);
+ mapRequest.setApplicationHomeSubCluster(applicationHomeSubCluster);
+ return mapRequest;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @return the mapping of the application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationHomeSubCluster getApplicationHomeSubCluster();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @param applicationHomeSubCluster the mapping of the application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..2145dd1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddApplicationHomeSubClusterResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * AddApplicationHomeSubClusterResponse contains the answer from the
+ * {@code FederationApplicationHomeSubClusterStore} to a request to insert a
+ * newly generated applicationId and its owner. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ *
+ */
+@Private
+@Unstable
+public abstract class AddApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static AddApplicationHomeSubClusterResponse newInstance() {
+ AddApplicationHomeSubClusterResponse response =
+ Records.newRecord(AddApplicationHomeSubClusterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
new file mode 100644
index 0000000..5e4c7cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * ApplicationHomeSubCluster is a report of the runtime information of the
+ * application that is running in the federated cluster.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link ApplicationId}</li>
+ * <li>{@link SubClusterId}</li>
+ * </ul>
+ *
+ */
+@Private
+@Unstable
+public abstract class ApplicationHomeSubCluster {
+
+ @Private
+ @Unstable
+ public static ApplicationHomeSubCluster newInstance(ApplicationId appId,
+ SubClusterId homeSubCluster) {
+ ApplicationHomeSubCluster appMapping =
+ Records.newRecord(ApplicationHomeSubCluster.class);
+ appMapping.setApplicationId(appId);
+ appMapping.setHomeSubCluster(homeSubCluster);
+ return appMapping;
+ }
+
+ /**
+ * Get the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @return the application identifier
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ /**
+ * Set the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @param applicationId the application identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+
+ /**
+ * Get the {@link SubClusterId} representing the unique identifier of the home
+ * subcluster in which the ApplicationMaster of the application is running.
+ *
+ * @return the home subcluster identifier
+ */
+ @Public
+ @Unstable
+ public abstract SubClusterId getHomeSubCluster();
+
+ /**
+ * Set the {@link SubClusterId} representing the unique identifier of the home
+ * subcluster in which the ApplicationMaster of the application is running.
+ *
+ * @param homeSubCluster the home subcluster identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setHomeSubCluster(SubClusterId homeSubCluster);
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ApplicationHomeSubCluster other = (ApplicationHomeSubCluster) obj;
+ if (!this.getApplicationId().equals(other.getApplicationId())) {
+ return false;
+ }
+ return this.getHomeSubCluster().equals(other.getHomeSubCluster());
+ }
+
+ @Override
+ public int hashCode() {
+ return getApplicationId().hashCode() * 31 + getHomeSubCluster().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "ApplicationHomeSubCluster [getApplicationId()="
+ + getApplicationId() + ", getHomeSubCluster()=" + getHomeSubCluster()
+ + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..f678aee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterRequest.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The request to <code>Federation state store</code> to delete the mapping of
+ * home subcluster of a submitted application.
+ */
+@Private
+@Unstable
+public abstract class DeleteApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static DeleteApplicationHomeSubClusterRequest newInstance(
+ ApplicationId applicationId) {
+ DeleteApplicationHomeSubClusterRequest deleteApplicationRequest =
+ Records.newRecord(DeleteApplicationHomeSubClusterRequest.class);
+ deleteApplicationRequest.setApplicationId(applicationId);
+ return deleteApplicationRequest;
+ }
+
+ /**
+ * Get the identifier of the {@link ApplicationId} to be removed from
+ * <code>Federation state store</code> .
+ *
+ * @return the identifier of the application to be removed from Federation
+ * State Store.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ /**
+ * Set the identifier of the {@link ApplicationId} to be removed from
+ * <code>Federation state store</code> .
+ *
+ * @param applicationId the identifier of the application to be removed from
+ * Federation State Store.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..fb1bef9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteApplicationHomeSubClusterResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * DeleteApplicationHomeSubClusterResponse contains the answer from the {@code
+ * FederationApplicationHomeSubClusterStore} to a request to delete the mapping
+ * of home subcluster of a submitted application. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ */
+@Private
+@Unstable
+public abstract class DeleteApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static DeleteApplicationHomeSubClusterResponse newInstance() {
+ DeleteApplicationHomeSubClusterResponse response =
+ Records.newRecord(DeleteApplicationHomeSubClusterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..a64d22e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterRequest.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain the home sub-cluster for the specified
+ * {@link ApplicationId}.
+ */
+@Private
+@Unstable
+public abstract class GetApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static GetApplicationHomeSubClusterRequest newInstance(
+ ApplicationId appId) {
+ GetApplicationHomeSubClusterRequest appMapping =
+ Records.newRecord(GetApplicationHomeSubClusterRequest.class);
+ appMapping.setApplicationId(appId);
+ return appMapping;
+ }
+
+ /**
+ * Get the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @return the application identifier
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationId getApplicationId();
+
+ /**
+ * Set the {@link ApplicationId} representing the unique identifier of the
+ * application.
+ *
+ * @param applicationId the application identifier
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationId(ApplicationId applicationId);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..60735b3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The response sent by <code>Federation state
+ * store</code> to a query for the home subcluster of a newly submitted
+ * application.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class GetApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static GetApplicationHomeSubClusterResponse newInstance(
+ ApplicationHomeSubCluster applicationHomeSubCluster) {
+ GetApplicationHomeSubClusterResponse mapResponse =
+ Records.newRecord(GetApplicationHomeSubClusterResponse.class);
+ mapResponse.setApplicationHomeSubCluster(applicationHomeSubCluster);
+ return mapResponse;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @return the mapping of the application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationHomeSubCluster getApplicationHomeSubCluster();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @param applicationHomeSubCluster the mapping of the application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
new file mode 100644
index 0000000..6054972
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain the home sub-cluster mapping of all active
+ * applications.
+ */
+@Private
+@Unstable
+public abstract class GetApplicationsHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static GetApplicationsHomeSubClusterRequest newInstance() {
+ GetApplicationsHomeSubClusterRequest request =
+ Records.newRecord(GetApplicationsHomeSubClusterRequest.class);
+ return request;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java
new file mode 100644
index 0000000..ba3d2c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterResponse.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The response sent by <code>Federation state
+ * store</code> to a query for the home subcluster of all submitted
+ * applications.
+ *
+ * <p>
+ * The response includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class GetApplicationsHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static GetApplicationsHomeSubClusterResponse newInstance(
+ List<ApplicationHomeSubCluster> appsHomeSubClusters) {
+ GetApplicationsHomeSubClusterResponse mapResponse =
+ Records.newRecord(GetApplicationsHomeSubClusterResponse.class);
+ mapResponse.setAppsHomeSubClusters(appsHomeSubClusters);
+ return mapResponse;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
+ *
+ * @return the mapping of all submitted application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract List<ApplicationHomeSubCluster> getAppsHomeSubClusters();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
+ *
+ * @param appsHomeSubClusters the mapping of all submitted application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setAppsHomeSubClusters(
+ List<ApplicationHomeSubCluster> appsHomeSubClusters);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java
new file mode 100644
index 0000000..eaa9252
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterRequest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent by the <code>Router</code> to
+ * <code>Federation state store</code> to update the home subcluster of a newly
+ * submitted application.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ApplicationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class UpdateApplicationHomeSubClusterRequest {
+
+ @Private
+ @Unstable
+ public static UpdateApplicationHomeSubClusterRequest newInstance(
+ ApplicationHomeSubCluster applicationHomeSubCluster) {
+ UpdateApplicationHomeSubClusterRequest updateApplicationRequest =
+ Records.newRecord(UpdateApplicationHomeSubClusterRequest.class);
+ updateApplicationRequest
+ .setApplicationHomeSubCluster(applicationHomeSubCluster);
+ return updateApplicationRequest;
+ }
+
+ /**
+ * Get the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @return the mapping of the application to it's home sub-cluster.
+ */
+ @Public
+ @Unstable
+ public abstract ApplicationHomeSubCluster getApplicationHomeSubCluster();
+
+ /**
+ * Set the {@link ApplicationHomeSubCluster} representing the mapping of the
+ * application to it's home sub-cluster.
+ *
+ * @param applicationHomeSubCluster the mapping of the application to it's
+ * home sub-cluster.
+ */
+ @Private
+ @Unstable
+ public abstract void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationHomeSubCluster);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java
new file mode 100644
index 0000000..7434335
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateApplicationHomeSubClusterResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * UpdateApplicationHomeSubClusterResponse contains the answer from the
+ * {@code FederationApplicationHomeSubClusterStore} to a request to register the
+ * home subcluster of a submitted application. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ */
+@Private
+@Unstable
+public abstract class UpdateApplicationHomeSubClusterResponse {
+
+ @Private
+ @Unstable
+ public static UpdateApplicationHomeSubClusterResponse newInstance() {
+ UpdateApplicationHomeSubClusterResponse response =
+ Records.newRecord(UpdateApplicationHomeSubClusterResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..2387cde
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link AddApplicationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class AddApplicationHomeSubClusterRequestPBImpl
+ extends AddApplicationHomeSubClusterRequest {
+
+ private AddApplicationHomeSubClusterRequestProto proto =
+ AddApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private AddApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public AddApplicationHomeSubClusterRequestPBImpl() {
+ builder = AddApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public AddApplicationHomeSubClusterRequestPBImpl(
+ AddApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public AddApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = AddApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationHomeSubCluster getApplicationHomeSubCluster() {
+ AddApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasAppSubclusterMap()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAppSubclusterMap());
+ }
+
+ @Override
+ public void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationInfo) {
+ maybeInitBuilder();
+ if (applicationInfo == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ builder.setAppSubclusterMap(convertToProtoFormat(applicationInfo));
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..1415981
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link AddApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class AddApplicationHomeSubClusterResponsePBImpl
+ extends AddApplicationHomeSubClusterResponse {
+
+ private AddApplicationHomeSubClusterResponseProto proto =
+ AddApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private AddApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public AddApplicationHomeSubClusterResponsePBImpl() {
+ builder = AddApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public AddApplicationHomeSubClusterResponsePBImpl(
+ AddApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public AddApplicationHomeSubClusterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
new file mode 100644
index 0000000..7e6a564
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link ApplicationHomeSubCluster}.
+ */
+@Private
+@Unstable
+public class ApplicationHomeSubClusterPBImpl extends ApplicationHomeSubCluster {
+
+ private ApplicationHomeSubClusterProto proto =
+ ApplicationHomeSubClusterProto.getDefaultInstance();
+ private ApplicationHomeSubClusterProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ApplicationId applicationId = null;
+ private SubClusterId homeSubCluster = null;
+
+ public ApplicationHomeSubClusterPBImpl() {
+ builder = ApplicationHomeSubClusterProto.newBuilder();
+ }
+
+ public ApplicationHomeSubClusterPBImpl(ApplicationHomeSubClusterProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ApplicationHomeSubClusterProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ApplicationHomeSubClusterProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null) {
+ builder.setApplicationId(convertToProtoFormat(this.applicationId));
+ }
+ if (this.homeSubCluster != null) {
+ builder.setHomeSubCluster(convertToProtoFormat(this.homeSubCluster));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ ApplicationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ this.applicationId = convertFromProtoFormat(p.getApplicationId());
+ return applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ return;
+ }
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ public SubClusterId getHomeSubCluster() {
+ ApplicationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.homeSubCluster != null) {
+ return this.homeSubCluster;
+ }
+ if (!p.hasHomeSubCluster()) {
+ return null;
+ }
+ this.homeSubCluster = convertFromProtoFormat(p.getHomeSubCluster());
+ return this.homeSubCluster;
+ }
+
+ @Override
+ public void setHomeSubCluster(SubClusterId homeSubCluster) {
+ maybeInitBuilder();
+ if (homeSubCluster == null) {
+ builder.clearHomeSubCluster();
+ }
+ this.homeSubCluster = homeSubCluster;
+ }
+
+ private SubClusterId convertFromProtoFormat(SubClusterIdProto subClusterId) {
+ return new SubClusterIdPBImpl(subClusterId);
+ }
+
+ private SubClusterIdProto convertToProtoFormat(SubClusterId subClusterId) {
+ return ((SubClusterIdPBImpl) subClusterId).getProto();
+ }
+
+ private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
+ return new ApplicationIdPBImpl(appId);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId appId) {
+ return ((ApplicationIdPBImpl) appId).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..b4ef680
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link DeleteApplicationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class DeleteApplicationHomeSubClusterRequestPBImpl
+ extends DeleteApplicationHomeSubClusterRequest {
+
+ private DeleteApplicationHomeSubClusterRequestProto proto =
+ DeleteApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private DeleteApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public DeleteApplicationHomeSubClusterRequestPBImpl() {
+ builder = DeleteApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public DeleteApplicationHomeSubClusterRequestPBImpl(
+ DeleteApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public DeleteApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = DeleteApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ DeleteApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getApplicationId());
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ return;
+ }
+ builder.setApplicationId(convertToProtoFormat(applicationId));
+ }
+
+ private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
+ return new ApplicationIdPBImpl(appId);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId appId) {
+ return ((ApplicationIdPBImpl) appId).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..8a37b3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link DeleteApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class DeleteApplicationHomeSubClusterResponsePBImpl
+ extends DeleteApplicationHomeSubClusterResponse {
+
+ private DeleteApplicationHomeSubClusterResponseProto proto =
+ DeleteApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private DeleteApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public DeleteApplicationHomeSubClusterResponsePBImpl() {
+ builder = DeleteApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public DeleteApplicationHomeSubClusterResponsePBImpl(
+ DeleteApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public DeleteApplicationHomeSubClusterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..865d0c4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class GetApplicationHomeSubClusterRequestPBImpl
+ extends GetApplicationHomeSubClusterRequest {
+
+ private GetApplicationHomeSubClusterRequestProto proto =
+ GetApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private GetApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private ApplicationId applicationId = null;
+
+ public GetApplicationHomeSubClusterRequestPBImpl() {
+ builder = GetApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public GetApplicationHomeSubClusterRequestPBImpl(
+ GetApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.applicationId != null) {
+ builder.setApplicationId(convertToProtoFormat(this.applicationId));
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ GetApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasApplicationId()) {
+ return null;
+ }
+ this.applicationId = convertFromProtoFormat(p.getApplicationId());
+ return applicationId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId applicationId) {
+ maybeInitBuilder();
+ if (applicationId == null) {
+ builder.clearApplicationId();
+ return;
+ }
+ this.applicationId = applicationId;
+ }
+
+ private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
+ return new ApplicationIdPBImpl(appId);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId appId) {
+ return ((ApplicationIdPBImpl) appId).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..1180488
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetApplicationHomeSubClusterResponsePBImpl
+ extends GetApplicationHomeSubClusterResponse {
+
+ private GetApplicationHomeSubClusterResponseProto proto =
+ GetApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private GetApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetApplicationHomeSubClusterResponsePBImpl() {
+ builder = GetApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public GetApplicationHomeSubClusterResponsePBImpl(
+ GetApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationHomeSubClusterResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetApplicationHomeSubClusterResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationHomeSubCluster getApplicationHomeSubCluster() {
+ GetApplicationHomeSubClusterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasAppSubclusterMap()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAppSubclusterMap());
+ }
+
+ @Override
+ public void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationInfo) {
+ maybeInitBuilder();
+ if (applicationInfo == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ builder.setAppSubclusterMap(convertToProtoFormat(applicationInfo));
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..3ce8d74
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationsHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class GetApplicationsHomeSubClusterRequestPBImpl
+ extends GetApplicationsHomeSubClusterRequest {
+
+ private GetApplicationsHomeSubClusterRequestProto proto =
+ GetApplicationsHomeSubClusterRequestProto.getDefaultInstance();
+ private GetApplicationsHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public GetApplicationsHomeSubClusterRequestPBImpl() {
+ builder = GetApplicationsHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public GetApplicationsHomeSubClusterRequestPBImpl(
+ GetApplicationsHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationsHomeSubClusterRequestProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-6211. Synchronization
improvement for moveApplicationAcrossQueues and updateApplicationPriority.
Contributed by Bibin A Chundatt.
Posted by su...@apache.org.
YARN-6211. Synchronization improvement for moveApplicationAcrossQueues and updateApplicationPriority. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a207aa99
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a207aa99
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a207aa99
Branch: refs/heads/YARN-2915
Commit: a207aa9930e7ee4f10228e2db4b4e733794eb8ea
Parents: 13d4bcf
Author: Sunil G <su...@apache.org>
Authored: Thu Feb 23 14:19:07 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Feb 23 14:19:07 2017 +0530
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/ClientRMService.java | 6 ++++--
.../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ++---
2 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a207aa99/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 0c87ede..48bccfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1228,7 +1228,8 @@ public class ClientRMService extends AbstractService implements
}
try {
- this.rmAppManager.moveApplicationAcrossQueue(applicationId,
+ this.rmAppManager.moveApplicationAcrossQueue(
+ application.getApplicationId(),
request.getTargetQueue());
} catch (YarnException ex) {
RMAuditLogger.logFailure(callerUGI.getShortUserName(),
@@ -1662,7 +1663,8 @@ public class ClientRMService extends AbstractService implements
}
try {
- rmAppManager.updateApplicationPriority(callerUGI, applicationId,
+ rmAppManager.updateApplicationPriority(callerUGI,
+ application.getApplicationId(),
newAppPriority);
} catch (YarnException ex) {
RMAuditLogger.logFailure(callerUGI.getShortUserName(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a207aa99/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index cc796e3..e211867 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRecoverEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -612,7 +611,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
RMApp app = this.rmContext.getRMApps().get(applicationId);
synchronized (applicationId) {
- if (app.isAppInCompletedStates()) {
+ if (app == null || app.isAppInCompletedStates()) {
return;
}
@@ -658,7 +657,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
// 2. Update this information to state-store
// 3. Perform real move operation and update in-memory data structures.
synchronized (applicationId) {
- if (app.isAppInCompletedStates()) {
+ if (app == null || app.isAppInCompletedStates()) {
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-5307. Federation Application
State Store internal APIs
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..8b72a1e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationsHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetApplicationsHomeSubClusterResponsePBImpl
+ extends GetApplicationsHomeSubClusterResponse {
+
+ private GetApplicationsHomeSubClusterResponseProto proto =
+ GetApplicationsHomeSubClusterResponseProto.getDefaultInstance();
+ private GetApplicationsHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List<ApplicationHomeSubCluster> appsHomeSubCluster;
+
+ public GetApplicationsHomeSubClusterResponsePBImpl() {
+ builder = GetApplicationsHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public GetApplicationsHomeSubClusterResponsePBImpl(
+ GetApplicationsHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetApplicationsHomeSubClusterResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetApplicationsHomeSubClusterResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.appsHomeSubCluster != null) {
+ addSubClustersInfoToProto();
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public List<ApplicationHomeSubCluster> getAppsHomeSubClusters() {
+ initSubClustersInfoList();
+ return appsHomeSubCluster;
+ }
+
+ @Override
+ public void setAppsHomeSubClusters(
+ List<ApplicationHomeSubCluster> appsHomeSubClusters) {
+ maybeInitBuilder();
+ if (appsHomeSubClusters == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ this.appsHomeSubCluster = appsHomeSubClusters;
+ }
+
+ private void initSubClustersInfoList() {
+ if (this.appsHomeSubCluster != null) {
+ return;
+ }
+ GetApplicationsHomeSubClusterResponseProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<ApplicationHomeSubClusterProto> subClusterInfosList =
+ p.getAppSubclusterMapList();
+ appsHomeSubCluster = new ArrayList<ApplicationHomeSubCluster>();
+
+ for (ApplicationHomeSubClusterProto r : subClusterInfosList) {
+ appsHomeSubCluster.add(convertFromProtoFormat(r));
+ }
+ }
+
+ private void addSubClustersInfoToProto() {
+ maybeInitBuilder();
+ builder.clearAppSubclusterMap();
+ if (appsHomeSubCluster == null) {
+ return;
+ }
+ Iterable<ApplicationHomeSubClusterProto> iterable =
+ new Iterable<ApplicationHomeSubClusterProto>() {
+ @Override
+ public Iterator<ApplicationHomeSubClusterProto> iterator() {
+ return new Iterator<ApplicationHomeSubClusterProto>() {
+
+ private Iterator<ApplicationHomeSubCluster> iter =
+ appsHomeSubCluster.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ApplicationHomeSubClusterProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+
+ }
+
+ };
+ builder.addAllAppSubclusterMap(iterable);
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
index d39ef7f..92bdf06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000..e42eb00
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link UpdateApplicationHomeSubClusterRequest} .
+ */
+@Private
+@Unstable
+public class UpdateApplicationHomeSubClusterRequestPBImpl
+ extends UpdateApplicationHomeSubClusterRequest {
+
+ private UpdateApplicationHomeSubClusterRequestProto proto =
+ UpdateApplicationHomeSubClusterRequestProto.getDefaultInstance();
+ private UpdateApplicationHomeSubClusterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public UpdateApplicationHomeSubClusterRequestPBImpl() {
+ builder = UpdateApplicationHomeSubClusterRequestProto.newBuilder();
+ }
+
+ public UpdateApplicationHomeSubClusterRequestPBImpl(
+ UpdateApplicationHomeSubClusterRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public UpdateApplicationHomeSubClusterRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = UpdateApplicationHomeSubClusterRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationHomeSubCluster getApplicationHomeSubCluster() {
+ UpdateApplicationHomeSubClusterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasAppSubclusterMap()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAppSubclusterMap());
+ }
+
+ @Override
+ public void setApplicationHomeSubCluster(
+ ApplicationHomeSubCluster applicationInfo) {
+ maybeInitBuilder();
+ if (applicationInfo == null) {
+ builder.clearAppSubclusterMap();
+ return;
+ }
+ builder.setAppSubclusterMap(convertToProtoFormat(applicationInfo));
+ }
+
+ private ApplicationHomeSubCluster convertFromProtoFormat(
+ ApplicationHomeSubClusterProto sc) {
+ return new ApplicationHomeSubClusterPBImpl(sc);
+ }
+
+ private ApplicationHomeSubClusterProto convertToProtoFormat(
+ ApplicationHomeSubCluster sc) {
+ return ((ApplicationHomeSubClusterPBImpl) sc).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000..ec31f0b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateApplicationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link UpdateApplicationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class UpdateApplicationHomeSubClusterResponsePBImpl
+ extends UpdateApplicationHomeSubClusterResponse {
+
+ private UpdateApplicationHomeSubClusterResponseProto proto =
+ UpdateApplicationHomeSubClusterResponseProto.getDefaultInstance();
+ private UpdateApplicationHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public UpdateApplicationHomeSubClusterResponsePBImpl() {
+ builder = UpdateApplicationHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public UpdateApplicationHomeSubClusterResponsePBImpl(
+ UpdateApplicationHomeSubClusterResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public UpdateApplicationHomeSubClusterResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 1b2e53e..b1ad310 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -90,4 +90,47 @@ message GetSubClustersInfoRequestProto {
message GetSubClustersInfoResponseProto {
repeated SubClusterInfoProto sub_cluster_infos = 1;
-}
\ No newline at end of file
+}
+
+message ApplicationHomeSubClusterProto {
+ optional ApplicationIdProto application_id = 1;
+ optional SubClusterIdProto home_sub_cluster = 2;
+}
+
+message AddApplicationHomeSubClusterRequestProto {
+ optional ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message AddApplicationHomeSubClusterResponseProto {
+}
+
+message UpdateApplicationHomeSubClusterRequestProto {
+ optional ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message UpdateApplicationHomeSubClusterResponseProto {
+}
+
+message GetApplicationHomeSubClusterRequestProto {
+ optional ApplicationIdProto application_id = 1;
+}
+
+message GetApplicationHomeSubClusterResponseProto {
+ optional ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message GetApplicationsHomeSubClusterRequestProto {
+
+}
+
+message GetApplicationsHomeSubClusterResponseProto {
+ repeated ApplicationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+
+message DeleteApplicationHomeSubClusterRequestProto {
+ optional ApplicationIdProto application_id = 1;
+}
+
+message DeleteApplicationHomeSubClusterResponseProto {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2d3d362/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
index 681edb1..210a246 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -19,6 +19,14 @@ package org.apache.hadoop.yarn.server.federation.store.records;
import org.apache.hadoop.yarn.api.BasePBImplRecordsTest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClusterInfoResponseProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetSubClustersInfoRequestProto;
@@ -31,6 +39,16 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClu
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterRequestProto;
import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationsHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetApplicationsHomeSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClusterInfoResponsePBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetSubClustersInfoRequestPBImpl;
@@ -43,6 +61,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubCluster
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterRequestPBImpl;
import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl;
import org.apache.hadoop.yarn.server.records.Version;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -58,6 +78,7 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
generateByNewInstance(Version.class);
generateByNewInstance(SubClusterId.class);
generateByNewInstance(SubClusterInfo.class);
+ generateByNewInstance(ApplicationHomeSubCluster.class);
}
@Test
@@ -130,4 +151,64 @@ public class TestFederationProtocolRecords extends BasePBImplRecordsTest {
GetSubClustersInfoResponseProto.class);
}
+ @Test
+ public void testAddApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(AddApplicationHomeSubClusterRequestPBImpl.class,
+ AddApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testAddApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(AddApplicationHomeSubClusterResponsePBImpl.class,
+ AddApplicationHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(UpdateApplicationHomeSubClusterRequestPBImpl.class,
+ UpdateApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testUpdateApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(UpdateApplicationHomeSubClusterResponsePBImpl.class,
+ UpdateApplicationHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(GetApplicationHomeSubClusterRequestPBImpl.class,
+ GetApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testGetApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(GetApplicationHomeSubClusterResponsePBImpl.class,
+ GetApplicationHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testGetApplicationsHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(GetApplicationsHomeSubClusterRequestPBImpl.class,
+ GetApplicationsHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testGetApplicationsHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(GetApplicationsHomeSubClusterResponsePBImpl.class,
+ GetApplicationsHomeSubClusterResponseProto.class);
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterRequest() throws Exception {
+ validatePBImplRecord(DeleteApplicationHomeSubClusterRequestPBImpl.class,
+ DeleteApplicationHomeSubClusterRequestProto.class);
+ }
+
+ @Test
+ public void testDeleteApplicationHomeSubClusterResponse() throws Exception {
+ validatePBImplRecord(DeleteApplicationHomeSubClusterResponsePBImpl.class,
+ DeleteApplicationHomeSubClusterResponseProto.class);
+ }
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org