You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2016/05/24 17:01:22 UTC
[1/5] incubator-slider git commit: SLIDER-1129 increment versions of
httpclient/http core dependencies; move to test scope only
Repository: incubator-slider
Updated Branches:
refs/heads/develop 5c391cc6c -> 44f4a5457
SLIDER-1129 increment versions of httpclient/http core dependencies; move to test scope only
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34f559a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34f559a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34f559a6
Branch: refs/heads/develop
Commit: 34f559a6716e429da5eda8ad17aab970fb7d6641
Parents: 5c391cc
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 19:37:04 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 19:37:04 2016 +0100
----------------------------------------------------------------------
pom.xml | 5 +++--
slider-core/pom.xml | 6 +-----
2 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f559a6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b616e15..36f16c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,8 @@
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
<httpclient.version>3.1</httpclient.version>
- <httpclient4.version>4.2.5</httpclient4.version>
+ <httpclient4.version>4.5.2</httpclient4.version>
+ <httpcore.version>4.4.4</httpcore.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -717,7 +718,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>${httpclient4.version}</version>
+ <version>${httpcore.version}</version>
</dependency>
<!-- ======================================================== -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f559a6/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 17728c3..6e942c0 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -335,11 +335,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
[5/5] incubator-slider git commit: Merge branch
'feature/SLIDER-1129-httpclient' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-1129-httpclient' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/44f4a545
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/44f4a545
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/44f4a545
Branch: refs/heads/develop
Commit: 44f4a5457a075a4315ae9a247daac5cadcb759cd
Parents: 5c391cc 80fe8c7
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 24 18:00:09 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 24 18:00:09 2016 +0100
----------------------------------------------------------------------
pom.xml | 32 ++-------
slider-core/pom.xml | 8 +--
.../appmaster/state/OutstandingRequest.java | 2 +
.../appstate/TestMockAppStateAAPlacement.groovy | 8 +--
.../appstate/TestMockLabelledAAPlacement.groovy | 11 +--
.../TestOutstandingRequestValidation.groovy | 26 -------
.../model/history/TestRoleHistoryAA.groovy | 12 ++--
...tRoleHistoryOutstandingRequestTracker.groovy | 5 +-
.../model/mock/MockContainerStatus.groovy | 34 ---------
.../appmaster/model/mock/MockFactory.groovy | 60 +++++++++++++++-
.../appmaster/model/mock/MockNodeReport.groovy | 75 --------------------
.../model/mock/MockRecordFactory.groovy | 37 ++++++++++
.../appmaster/model/mock/MockYarnCluster.groovy | 9 ++-
.../appmaster/model/mock/MockYarnEngine.groovy | 5 +-
14 files changed, 131 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
[3/5] incubator-slider git commit: SLIDER-1130 Hadoop 2.8 YARN RPC
changes have broken slider mock classes
Posted by st...@apache.org.
SLIDER-1130 Hadoop 2.8 YARN RPC changes have broken slider mock classes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9937bbca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9937bbca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9937bbca
Branch: refs/heads/develop
Commit: 9937bbcae3ac1575a82b0140558264f2507962ff
Parents: 1154470
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 22:40:49 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 22:40:49 2016 +0100
----------------------------------------------------------------------
.../appstate/TestMockAppStateAAPlacement.groovy | 4 +-
.../appstate/TestMockLabelledAAPlacement.groovy | 11 +--
.../model/history/TestRoleHistoryAA.groovy | 12 ++--
.../model/mock/MockContainerStatus.groovy | 34 ---------
.../appmaster/model/mock/MockFactory.groovy | 60 +++++++++++++++-
.../appmaster/model/mock/MockNodeReport.groovy | 75 --------------------
.../model/mock/MockRecordFactory.groovy | 37 ++++++++++
.../appmaster/model/mock/MockYarnCluster.groovy | 9 ++-
.../appmaster/model/mock/MockYarnEngine.groovy | 5 +-
9 files changed, 114 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index e43d894..85c09ff 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -24,12 +24,10 @@ import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -283,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index f0fed95..b2e2a74 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -22,7 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
super.setup()
// node 1 is GPU
- updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
- updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
@@ -108,7 +108,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
assert 0 == appState.reviewRequestAndReleaseNodes().size()
// switch node 2 into being labelled
- def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu"))
+ def outcome = updateNodes(MockFactory.instance.
+ newNodeReport("00000002", NodeState.RUNNING, "gpu"))
assert cloneNodemap().size() == NODES
assert outcome.clusterChanged
@@ -118,7 +119,7 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("00000004", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
index db84b0b..9c60ea5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -21,12 +21,9 @@ package org.apache.slider.server.appmaster.model.history
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.api.proto.Messages
import org.apache.slider.api.proto.RestTypeMarshalling
-import org.apache.slider.api.types.NodeInformation
import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -72,7 +69,7 @@ class TestRoleHistoryAA extends SliderTestBase {
}
protected boolean setNodeState(NodeInstance node, NodeState state) {
- node.updateNode(new MockNodeReport(node.hostname, state))
+ node.updateNode(MockFactory.instance.newNodeReport(node.hostname, state))
}
@Test
@@ -84,7 +81,8 @@ class TestRoleHistoryAA extends SliderTestBase {
@Test
public void testFindSomeNodesSomeLabel() throws Throwable {
// all three will surface at first
- update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")])
+ update(nodeMap,
+ [MockFactory.instance.newNodeReport("1", NodeState.RUNNING, "GPU")])
def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU")
verifyResultSize(1, gpuNodes)
def instance = gpuNodes[0]
@@ -159,7 +157,7 @@ class TestRoleHistoryAA extends SliderTestBase {
def role1 = node1.getOrCreate(1)
def role2 = node1.getOrCreate(2)
nodeMap.values().each {
- it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY))
+ it.updateNode(MockFactory.instance.newNodeReport("0", NodeState.UNHEALTHY))
}
assertNoAvailableNodes(1)
assertNoAvailableNodes(2)
@@ -249,6 +247,6 @@ class TestRoleHistoryAA extends SliderTestBase {
def NodeMap createNodeMap(List<String> hosts, NodeState state,
String label = "") {
- createNodeMap(MockNodeReport.createInstances(hosts, state, label))
+ createNodeMap(MockFactory.instance.createNodeReports(hosts, state, label))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
deleted file mode 100644
index 4237f1e..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.hadoop.yarn.api.records.ContainerState
-import org.apache.hadoop.yarn.api.records.ContainerStatus
-import org.apache.hadoop.yarn.api.records.Resource
-
-class MockContainerStatus extends ContainerStatus {
-
- ContainerId containerId
- ContainerState state
- String diagnostics
- int exitStatus
-
- Resource capability;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index d873390..3ba6e31 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -23,8 +23,13 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
@@ -40,6 +45,8 @@ import org.apache.slider.providers.ProviderRole
@Slf4j
class MockFactory implements MockRoles {
+ public static MockFactory instance = new MockFactory();
+
/*
Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
*/
@@ -239,7 +246,56 @@ class MockFactory implements MockRoles {
return new MockResource(memory, vcores)
}
- MockContainerStatus newContainerStatus() {
- return new MockContainerStatus()
+ ContainerStatus newContainerStatus() {
+ return newContainerStatus(null, null, "", 0)
+ }
+
+ ContainerStatus newContainerStatus(ContainerId containerId,
+ ContainerState containerState, String diagnostics, int exitStatus) {
+ ContainerStatus.newInstance(containerId, containerState, diagnostics, exitStatus)
+ }
+
+ /**
+ * Create a single instance
+ * @param hostname
+ * @param nodeState
+ * @param label
+ */
+ NodeReport newNodeReport(String hostname, NodeState nodeState, String label = "") {
+ def nodeId = NodeId.newInstance(hostname, 80)
+ Integer.valueOf(hostname, 16)
+ newNodeReport(hostname, nodeId, nodeState, label)
}
+
+ NodeReport newNodeReport(
+ String hostname,
+ NodeId nodeId,
+ NodeState nodeState,
+ String label) {
+ def report = new NodeReportPBImpl();
+ def nodeLabels = new HashSet<>()
+ nodeLabels.add(label)
+ report.nodeId = nodeId
+ report.nodeLabels = nodeLabels
+ report.nodeState = nodeState
+ report.httpAddress = "http$hostname:80"
+ report
+ }
+
+ /**
+ * Create a list of instances -one for each hostname
+ * @param hostnames hosts
+ * @param nodeState state of all of them
+ * @param label label for all of them
+ * @return
+ */
+ List<NodeReport> createNodeReports(
+ List<String> hostnames,
+ NodeState nodeState = NodeState.RUNNING,
+ String label = "") {
+ hostnames.collect { String name ->
+ newNodeReport(name, nodeState, label)
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
deleted file mode 100644
index 8c3b712..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import groovy.transform.CompileStatic
-import org.apache.hadoop.yarn.api.records.NodeId
-import org.apache.hadoop.yarn.api.records.NodeReport
-import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.hadoop.yarn.api.records.Resource
-
-/**
- * Node report for testing
- */
-@CompileStatic
-class MockNodeReport extends NodeReport {
- NodeId nodeId;
- NodeState nodeState;
- String httpAddress;
- String rackName;
- Resource used;
- Resource capability;
- int numContainers;
- String healthReport;
- long lastHealthReportTime;
- Set<String> nodeLabels;
-
- MockNodeReport() {
- }
-
- /**
- * Create a single instance
- * @param hostname
- * @param nodeState
- * @param label
- */
- MockNodeReport(String hostname, NodeState nodeState, String label ="") {
- nodeId = NodeId.newInstance(hostname, 80)
- Integer.valueOf(hostname, 16)
- this.nodeState = nodeState
- this.httpAddress = "http$hostname:80"
- this.nodeLabels = new HashSet<>()
- nodeLabels.add(label)
- }
-
- /**
- * Create a list of instances -one for each hostname
- * @param hostnames hosts
- * @param nodeState state of all of them
- * @param label label for all of them
- * @return
- */
- static List<MockNodeReport> createInstances(
- List<String> hostnames,
- NodeState nodeState = NodeState.RUNNING,
- String label = "") {
- hostnames.collect { String name ->
- new MockNodeReport(name, nodeState, label)}
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
new file mode 100644
index 0000000..a608e5a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
+
+/**
+ * Node report for testing
+ */
+@CompileStatic
+class MockRecordFactory {
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 265a796..e1dfa25 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
/**
@@ -149,7 +150,7 @@ public class MockYarnCluster {
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
nodes.collect { MockYarnClusterNode n -> n.nodeReport }
}
@@ -167,7 +168,7 @@ public class MockYarnCluster {
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
- public MockNodeReport nodeReport
+ public NodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -181,9 +182,7 @@ public class MockYarnCluster {
containers[i] = new MockYarnClusterContainer(mci)
}
- nodeReport = new MockNodeReport()
- nodeReport.nodeId = nodeId
- nodeReport.nodeState = NodeState.RUNNING
+ nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 7ab97fa..d946c6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
@@ -166,7 +167,7 @@ class MockYarnEngine {
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
cluster.nodeReports
}
-}
\ No newline at end of file
+}
[2/5] incubator-slider git commit: SLIDER-1129 remove all httpclient
maven imports: use the one the Hadoop version gives you
Posted by st...@apache.org.
SLIDER-1129 remove all httpclient maven imports: use the one the Hadoop version gives you
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1154470d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1154470d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1154470d
Branch: refs/heads/develop
Commit: 1154470d33e1eabde81e020119145996949e0df9
Parents: 34f559a
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 22:40:16 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 22:40:16 2016 +0100
----------------------------------------------------------------------
pom.xml | 31 +++++--------------------------
slider-core/pom.xml | 2 ++
2 files changed, 7 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1154470d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 36f16c0..b1f2000 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,8 +154,6 @@
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
<httpclient.version>3.1</httpclient.version>
- <httpclient4.version>4.5.2</httpclient4.version>
- <httpcore.version>4.4.4</httpcore.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -522,12 +520,6 @@
</exclusions>
</dependency>
- <!-- hadoop-client includes the following jars, so they do not need to be
- included separately:
- hadoop-common, hadoop-hdfs (client?), hadoop-mapreduce-client-app,
- hadoop-yarn-api, hadoop-mapreduce-client-core,
- hadoop-mapreduce-client-jobclient, and hadoop-annotations
- -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -541,14 +533,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -569,14 +553,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -697,6 +673,8 @@
<version>${commons-lang.version}</version>
</dependency>
+<!--
+
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
@@ -708,7 +686,7 @@
</exclusion>
</exclusions>
</dependency>
-
+ T
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -720,7 +698,8 @@
<artifactId>httpcore</artifactId>
<version>${httpcore.version}</version>
</dependency>
-
+-->
+
<!-- ======================================================== -->
<!-- HBASE -->
<!-- ======================================================== -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1154470d/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 6e942c0..8b921a5 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -326,6 +326,7 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+<!--
<dependency>
<groupId>commons-httpclient</groupId>
@@ -337,6 +338,7 @@
<artifactId>httpclient</artifactId>
<scope>test</scope>
</dependency>
+-->
<dependency>
<groupId>commons-lang</groupId>
[4/5] incubator-slider git commit: SLIDER-1130 Hadoop 2.8 YARN
changes have broken slider. See also YARN-5135: ContainerRequest labels
ignored -broken constructor
Posted by st...@apache.org.
SLIDER-1130 Hadoop 2.8 YARN changes have broken slider. See also YARN-5135: ContainerRequest labels ignored -broken constructor
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80fe8c7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80fe8c7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80fe8c7e
Branch: refs/heads/develop
Commit: 80fe8c7e04a933885ab0f623216940861b233b5f
Parents: 9937bbc
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 24 17:59:58 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 24 17:59:58 2016 +0100
----------------------------------------------------------------------
.../appmaster/state/OutstandingRequest.java | 2 ++
.../appstate/TestMockAppStateAAPlacement.groovy | 6 ++---
.../appstate/TestMockLabelledAAPlacement.groovy | 4 +--
.../TestOutstandingRequestValidation.groovy | 26 --------------------
...tRoleHistoryOutstandingRequestTracker.groovy | 5 +++-
.../appmaster/model/mock/MockYarnCluster.groovy | 2 +-
6 files changed, 12 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index e0b01a1..c6524be 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -283,6 +283,7 @@ public final class OutstandingRequest extends RoleHostnamePair {
*/
public synchronized AMRMClient.ContainerRequest escalate() {
Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued " + this);
+ log.debug("Escalating {}", this.toString());
escalated = true;
// this is now the priority
@@ -352,6 +353,7 @@ public final class OutstandingRequest extends RoleHostnamePair {
}
sb.append(", node=").append(node);
sb.append(", hasLocation=").append(requestHasLocation);
+ sb.append(", label=").append(label);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
sb.append(", mayEscalate=").append(mayEscalate);
sb.append(", escalated=").append(escalated);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 85c09ff..125b1f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -57,7 +57,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
/**
- * This is the simplest AA allocation: no lables, so allocate anywhere
+ * This is the simplest AA allocation: no labels, so allocate anywhere
* @throws Throwable
*/
@Test
@@ -218,7 +218,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
// flex down so that the next request should be cancelled
aaRole.desired = 0
- // expect: no new reqests, pending count --
+ // expect: no new requests, pending count --
List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
assert aaRole.pendingAntiAffineRequests == 0
assert !aaRole.AARequestOutstanding
@@ -281,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("4", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index b2e2a74..e159dbb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
super.setup()
// node 1 is GPU
- updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
- updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.instance.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.instance.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
index 9202b5c..fb9d976 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
@@ -55,21 +55,6 @@ class TestOutstandingRequestValidation extends SliderTestBase {
createAndValidate(H1, "", true)
}
- @Test
- public void testLabelAndHostRelaxed() throws Throwable {
- expectValidationFailure(H1, "gpu", true)
- }
-
- @Test
- public void testLabelAndHostNonRelaxed() throws Throwable {
- expectValidationFailure(H1, "gpu", false)
- }
-
- @Test
- public void testComplexLabelExpression() throws Throwable {
- expectValidationFailure(null, "gpu||ps4", true)
- }
-
/**
* Use varargs for simple list to array conversion
* @param hostnames host names
@@ -79,17 +64,6 @@ class TestOutstandingRequestValidation extends SliderTestBase {
hostnames
}
- void expectValidationFailure(
- String[] hosts,
- String labels,
- boolean relaxLocality) {
- try {
- def result = createAndValidate(hosts, labels, relaxLocality)
- fail("Expected an exception, got $result")
- } catch (InvalidContainerRequestException expected) {
- }
- }
-
void expectCreationFailure(
String[] hosts,
String labels,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 6969b38..0969824 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -238,12 +238,15 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
def workerRole = lookupRole(WORKER.name)
// initial request
def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0)
+ assert (req1.label == WORKERS_LABEL)
+
assert (yarnRequest.nodeLabelExpression == null)
assert (!yarnRequest.relaxLocality)
// escalation
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
+ assert yarnRequest2.nodes == null
assert (yarnRequest2.relaxLocality)
+ assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index e1dfa25..f0a1494 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -182,7 +182,7 @@ public class MockYarnCluster {
containers[i] = new MockYarnClusterContainer(mci)
}
- nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
+ nodeReport = MockFactory.instance.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
}
/**