You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by el...@apache.org on 2016/06/18 23:16:06 UTC
[16/31] incubator-slider git commit: SLIDER-1130 Hadoop 2.8 YARN RPC
changes have broken slider mock classes
SLIDER-1130 Hadoop 2.8 YARN RPC changes have broken slider mock classes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9937bbca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9937bbca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9937bbca
Branch: refs/heads/branches/branch-0.91
Commit: 9937bbcae3ac1575a82b0140558264f2507962ff
Parents: 1154470
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 22:40:49 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 22:40:49 2016 +0100
----------------------------------------------------------------------
.../appstate/TestMockAppStateAAPlacement.groovy | 4 +-
.../appstate/TestMockLabelledAAPlacement.groovy | 11 +--
.../model/history/TestRoleHistoryAA.groovy | 12 ++--
.../model/mock/MockContainerStatus.groovy | 34 ---------
.../appmaster/model/mock/MockFactory.groovy | 60 +++++++++++++++-
.../appmaster/model/mock/MockNodeReport.groovy | 75 --------------------
.../model/mock/MockRecordFactory.groovy | 37 ++++++++++
.../appmaster/model/mock/MockYarnCluster.groovy | 9 ++-
.../appmaster/model/mock/MockYarnEngine.groovy | 5 +-
9 files changed, 114 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index e43d894..85c09ff 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -24,12 +24,10 @@ import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -283,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index f0fed95..b2e2a74 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -22,7 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
super.setup()
// node 1 is GPU
- updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
- updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
@@ -108,7 +108,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
assert 0 == appState.reviewRequestAndReleaseNodes().size()
// switch node 2 into being labelled
- def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu"))
+ def outcome = updateNodes(MockFactory.instance.
+ newNodeReport("00000002", NodeState.RUNNING, "gpu"))
assert cloneNodemap().size() == NODES
assert outcome.clusterChanged
@@ -118,7 +119,7 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("00000004", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
index db84b0b..9c60ea5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -21,12 +21,9 @@ package org.apache.slider.server.appmaster.model.history
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.api.proto.Messages
import org.apache.slider.api.proto.RestTypeMarshalling
-import org.apache.slider.api.types.NodeInformation
import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -72,7 +69,7 @@ class TestRoleHistoryAA extends SliderTestBase {
}
protected boolean setNodeState(NodeInstance node, NodeState state) {
- node.updateNode(new MockNodeReport(node.hostname, state))
+ node.updateNode(MockFactory.instance.newNodeReport(node.hostname, state))
}
@Test
@@ -84,7 +81,8 @@ class TestRoleHistoryAA extends SliderTestBase {
@Test
public void testFindSomeNodesSomeLabel() throws Throwable {
// all three will surface at first
- update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")])
+ update(nodeMap,
+ [MockFactory.instance.newNodeReport("1", NodeState.RUNNING, "GPU")])
def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU")
verifyResultSize(1, gpuNodes)
def instance = gpuNodes[0]
@@ -159,7 +157,7 @@ class TestRoleHistoryAA extends SliderTestBase {
def role1 = node1.getOrCreate(1)
def role2 = node1.getOrCreate(2)
nodeMap.values().each {
- it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY))
+ it.updateNode(MockFactory.instance.newNodeReport("0", NodeState.UNHEALTHY))
}
assertNoAvailableNodes(1)
assertNoAvailableNodes(2)
@@ -249,6 +247,6 @@ class TestRoleHistoryAA extends SliderTestBase {
def NodeMap createNodeMap(List<String> hosts, NodeState state,
String label = "") {
- createNodeMap(MockNodeReport.createInstances(hosts, state, label))
+ createNodeMap(MockFactory.instance.createNodeReports(hosts, state, label))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
deleted file mode 100644
index 4237f1e..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.hadoop.yarn.api.records.ContainerState
-import org.apache.hadoop.yarn.api.records.ContainerStatus
-import org.apache.hadoop.yarn.api.records.Resource
-
-class MockContainerStatus extends ContainerStatus {
-
- ContainerId containerId
- ContainerState state
- String diagnostics
- int exitStatus
-
- Resource capability;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index d873390..3ba6e31 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -23,8 +23,13 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
@@ -40,6 +45,8 @@ import org.apache.slider.providers.ProviderRole
@Slf4j
class MockFactory implements MockRoles {
+ public static MockFactory instance = new MockFactory();
+
/*
Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
*/
@@ -239,7 +246,56 @@ class MockFactory implements MockRoles {
return new MockResource(memory, vcores)
}
- MockContainerStatus newContainerStatus() {
- return new MockContainerStatus()
+ ContainerStatus newContainerStatus() {
+ return newContainerStatus(null, null, "", 0)
+ }
+
+ ContainerStatus newContainerStatus(ContainerId containerId,
+ ContainerState containerState, String diagnostics, int exitStatus) {
+ ContainerStatus.newInstance(containerId, containerState, diagnostics, exitStatus)
+ }
+
+ /**
+ * Create a single instance
+ * @param hostname
+ * @param nodeState
+ * @param label
+ */
+ NodeReport newNodeReport(String hostname, NodeState nodeState, String label = "") {
+ def nodeId = NodeId.newInstance(hostname, 80)
+ Integer.valueOf(hostname, 16)
+ newNodeReport(hostname, nodeId, nodeState, label)
}
+
+ NodeReport newNodeReport(
+ String hostname,
+ NodeId nodeId,
+ NodeState nodeState,
+ String label) {
+ def report = new NodeReportPBImpl();
+ def nodeLabels = new HashSet<>()
+ nodeLabels.add(label)
+ report.nodeId = nodeId
+ report.nodeLabels = nodeLabels
+ report.nodeState = nodeState
+ report.httpAddress = "http$hostname:80"
+ report
+ }
+
+ /**
+ * Create a list of instances -one for each hostname
+ * @param hostnames hosts
+ * @param nodeState state of all of them
+ * @param label label for all of them
+ * @return
+ */
+ List<NodeReport> createNodeReports(
+ List<String> hostnames,
+ NodeState nodeState = NodeState.RUNNING,
+ String label = "") {
+ hostnames.collect { String name ->
+ newNodeReport(name, nodeState, label)
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
deleted file mode 100644
index 8c3b712..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import groovy.transform.CompileStatic
-import org.apache.hadoop.yarn.api.records.NodeId
-import org.apache.hadoop.yarn.api.records.NodeReport
-import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.hadoop.yarn.api.records.Resource
-
-/**
- * Node report for testing
- */
-@CompileStatic
-class MockNodeReport extends NodeReport {
- NodeId nodeId;
- NodeState nodeState;
- String httpAddress;
- String rackName;
- Resource used;
- Resource capability;
- int numContainers;
- String healthReport;
- long lastHealthReportTime;
- Set<String> nodeLabels;
-
- MockNodeReport() {
- }
-
- /**
- * Create a single instance
- * @param hostname
- * @param nodeState
- * @param label
- */
- MockNodeReport(String hostname, NodeState nodeState, String label ="") {
- nodeId = NodeId.newInstance(hostname, 80)
- Integer.valueOf(hostname, 16)
- this.nodeState = nodeState
- this.httpAddress = "http$hostname:80"
- this.nodeLabels = new HashSet<>()
- nodeLabels.add(label)
- }
-
- /**
- * Create a list of instances -one for each hostname
- * @param hostnames hosts
- * @param nodeState state of all of them
- * @param label label for all of them
- * @return
- */
- static List<MockNodeReport> createInstances(
- List<String> hostnames,
- NodeState nodeState = NodeState.RUNNING,
- String label = "") {
- hostnames.collect { String name ->
- new MockNodeReport(name, nodeState, label)}
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
new file mode 100644
index 0000000..a608e5a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
+
+/**
+ * Node report for testing
+ */
+@CompileStatic
+class MockRecordFactory {
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 265a796..e1dfa25 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
/**
@@ -149,7 +150,7 @@ public class MockYarnCluster {
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
nodes.collect { MockYarnClusterNode n -> n.nodeReport }
}
@@ -167,7 +168,7 @@ public class MockYarnCluster {
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
- public MockNodeReport nodeReport
+ public NodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -181,9 +182,7 @@ public class MockYarnCluster {
containers[i] = new MockYarnClusterContainer(mci)
}
- nodeReport = new MockNodeReport()
- nodeReport.nodeId = nodeId
- nodeReport.nodeState = NodeState.RUNNING
+ nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 7ab97fa..d946c6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
@@ -166,7 +167,7 @@ class MockYarnEngine {
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
cluster.nodeReports
}
-}
\ No newline at end of file
+}