You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/17 22:12:51 UTC
[1/8] incubator-slider git commit: need to provide details on pending
AA requests for Web UI to be accurate
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-82-pass-3.1 8c5065d92 -> 57638ff4e
need to provide details on pending AA requests for Web UI to be accurate
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7278c39c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7278c39c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7278c39c
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 7278c39c9bf85c7a217ce5568760f6738cb34a2e
Parents: 8c5065d
Author: Steve Loughran <st...@apache.org>
Authored: Sun Nov 15 19:19:52 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Nov 15 19:19:52 2015 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7278c39c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 4a3cc45..d977323 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -243,13 +243,19 @@ public class AppState {
*/
private final LongGauge surplusContainers = new LongGauge();
-
/**
- * Track the number of requested Containers
+ * Track the number of requested containers.
+ * Important: this does not include AA requests which are yet to be issued.
*/
private final LongGauge outstandingContainerRequests = new LongGauge();
/**
+ * Track the number of pending (not yet active) requests
+ * Important: this does not include AA requests which are yet to be issued.
+ */
+ private final LongGauge pendingAARequests = new LongGauge();
+
+ /**
* Map of requested nodes. This records the command used to start it,
* resources, etc. When container started callback is received,
* the node is promoted from here to the containerMap
@@ -376,6 +382,10 @@ public class AppState {
startFailedContainerCount.inc();
}
+ public long getTotalOutstandingRequests() {
+ return outstandingContainerRequests.get() +
+ pendingAARequests.get();
+ }
public AtomicInteger getCompletionOfNodeNotInLiveListEvent() {
return completionOfNodeNotInLiveListEvent;
}
@@ -405,7 +415,6 @@ public class AppState {
return completedContainers;
}
-
public Map<ContainerId, RoleInstance> getFailedContainers() {
return failedContainers;
}
[7/8] incubator-slider git commit: SLIDER-989 Mock AA test for
nodemap not updated
Posted by st...@apache.org.
SLIDER-989 Mock AA test for nodemap not updated
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e71ffbdb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e71ffbdb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e71ffbdb
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: e71ffbdb943c8bb3724d130be71bec7be238ed66
Parents: b2b58d3
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 17 20:10:13 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 17 20:10:13 2015 +0000
----------------------------------------------------------------------
.../server/appmaster/state/AppStateBindingInfo.java | 2 +-
.../model/appstate/TestMockAppStateAAOvercapacity.groovy | 8 --------
.../model/appstate/TestMockAppStateAAPlacement.groovy | 11 +++++++++++
.../appmaster/model/mock/BaseMockAppStateTest.groovy | 3 +--
4 files changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e71ffbdb/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
index a4a9b7e..a8aa1a2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -47,7 +47,7 @@ public class AppStateBindingInfo {
public List<Container> liveContainers = new ArrayList<>(0);
public Map<String, String> applicationInfo = new HashMap<>();
public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
- /** node reports off the RM. If null, the app state needs to be given a node update later */
+ /** node reports off the RM. */
public List<NodeReport> nodeReports = new ArrayList<>(0);
public void validate() throws IllegalArgumentException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e71ffbdb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
index 7728748..a8c50d1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
@@ -21,19 +21,11 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.core.main.LauncherExitCodes
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.ContainerAssignment
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.NodeMap
-import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e71ffbdb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 3461e23..cdfa9e2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
@@ -271,4 +272,14 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
assert 1 == appState.reviewRequestAndReleaseNodes().size()
}
+ @Test
+ public void testBindingInfoMustHaveNodeMap() throws Throwable {
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.nodeReports = null;
+ try {
+ def state = new MockAppState(bindingInfo)
+ fail("Expected an exception, got $state")
+ } catch (IllegalArgumentException expected) {
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e71ffbdb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index a53e0be..69a98eb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -96,8 +96,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
- appState = new MockAppState()
- appState.buildInstance(buildBindingInfo())
+ appState = new MockAppState(buildBindingInfo())
stateAccess = new ProviderAppState(testName, appState)
}
[8/8] incubator-slider git commit: SLIDER-969 Add mock test to
simulate AA failure and restart of AA request sequence
Posted by st...@apache.org.
SLIDER-969 Add mock test to simulate AA failure and restart of AA request sequence
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/57638ff4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/57638ff4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/57638ff4
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 57638ff4e6ca24173106b0df98f512c6205fc869
Parents: e71ffbd
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 17 21:12:54 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 17 21:12:54 2015 +0000
----------------------------------------------------------------------
.../appstate/TestMockAppStateAAPlacement.groovy | 29 ++++++++++++++++++++
.../TestMockAppStateRebuildOnAMRestart.groovy | 14 ++--------
2 files changed, 32 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57638ff4/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index cdfa9e2..4eff059 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -23,7 +23,11 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
@@ -282,4 +286,29 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
} catch (IllegalArgumentException expected) {
}
}
+
+ @Test
+ public void testAMRestart() throws Throwable {
+ def desiredAA = 3
+ aaRole.desired = desiredAA
+ List<RoleInstance> instances = createAndStartNodes()
+ List<Container> containers = instances.collect { it.container }
+
+ // now destroy the app state
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.instanceDefinition = factory.newInstanceDefinition(0, 0, desiredAA)
+ ConfTreeOperations cto = new ConfTreeOperations(bindingInfo.instanceDefinition.resources)
+ cto.setComponentOpt(ROLE2,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED)
+ bindingInfo.liveContainers = containers
+ appState = new MockAppState(bindingInfo)
+
+ def aaRole = lookupRole(MockFactory.AAROLE_2.name)
+ def gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name)
+ appState.reviewRequestAndReleaseNodes()
+ assert aaRole.isAntiAffinePlacement()
+ assert aaRole.isAARequestOutstanding()
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57638ff4/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index 59cc2c8..0e526b6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -58,23 +58,15 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest implements
assert instances.size() == clusterSize
//clone the list
- List<RoleInstance> cloned = [];
- List<Container> containers = []
- instances.each { RoleInstance elt ->
- cloned.add(elt.clone() as RoleInstance)
- containers.add(elt.container)
- }
+ List<Container> containers = instances.collect { it.container }
NodeMap nodemap = appState.roleHistory.cloneNodemap()
- // now destroy the app state
- appState = new MockAppState()
-
//and rebuild
def bindingInfo = buildBindingInfo()
bindingInfo.instanceDefinition = factory.newInstanceDefinition(r0, r1, r2)
bindingInfo.liveContainers = containers
- appState.buildInstance(bindingInfo)
+ appState = new MockAppState(bindingInfo)
assert appState.startedCountainerCount == clusterSize
@@ -107,7 +99,7 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest implements
}
assert 0 == appState.reviewRequestAndReleaseNodes().size()
- def status = appState.getClusterStatus()
+ def status = appState.clusterStatus
// verify the AM restart container count was set
String restarted = status.getInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null;
[3/8] incubator-slider git commit: SLIDER-979 AM web UI to show state
of AA request
Posted by st...@apache.org.
SLIDER-979 AM web UI to show state of AA request
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8efc4c2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8efc4c2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8efc4c2c
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 8efc4c2c5e7af89ffd36af25159c1ac5afad2759
Parents: c776b1a
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 16 16:12:10 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 16 16:12:10 2015 +0000
----------------------------------------------------------------------
.../server/appmaster/management/LongGauge.java | 6 ++
.../slider/server/appmaster/state/AppState.java | 3 +-
.../server/appmaster/web/view/IndexBlock.java | 9 ++-
.../providers/agent/DemoAgentAAEcho.groovy | 45 +++++++++++
.../providers/agent/TestAgentAAEcho.groovy | 9 ++-
.../appstate/BaseMockAppStateAATest.groovy | 62 ++++++++++++++++
.../appstate/TestMockAppStateAAPlacement.groovy | 41 +---------
.../appstate/TestMockLabelledAAPlacement.groovy | 55 ++++----------
.../appmaster/model/mock/MockFactory.groovy | 37 +++++++++-
.../appmaster/model/mock/MockRoles.groovy | 2 +
.../appmaster/web/view/TestIndexBlock.groovy | 78 +++++++++++++++-----
.../slider/server/management/TestGauges.groovy | 52 +++++++++++++
.../apache/slider/test/SliderTestUtils.groovy | 45 ++++++++---
13 files changed, 324 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
index ac9ac0e..c93467b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
@@ -56,6 +56,11 @@ public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
return get();
}
+ /**
+ * Method from {@Code counter}; used here for drop-in replacement
+ * without any recompile
+ * @return current value
+ */
public Long getCount() {
return get();
}
@@ -66,6 +71,7 @@ public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
public void inc() {
incrementAndGet();
}
+
/**
* {@code --}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index d977323..4152a89 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1231,13 +1231,14 @@ public class AppState {
* @return the container request to submit or null if there is none
*/
private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) {
- incrementRequestCount(role);
if (role.isAntiAffinePlacement()) {
return createAAContainerRequest(role);
} else {
+ incrementRequestCount(role);
return roleHistory.requestContainerForRole(role).getIssuedRequest();
}
}
+
/**
* Create a container request.
* Update internal state, such as the role request count.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 5131b5e..41e1c01 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -45,6 +45,13 @@ import java.util.Map.Entry;
public class IndexBlock extends HtmlBlock {
private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
+ /**
+ * Message printed when application is at full size.
+ *
+ * {@value}
+ */
+ public static final String ALL_CONTAINERS_ALLOCATED = "all containers allocated";
+
private StateAccessForProviders appView;
private ProviderService providerService;
@@ -77,7 +84,7 @@ public class IndexBlock extends HtmlBlock {
appView.getApplicationLivenessInformation();
String livestatus =
liveness.allRequestsSatisfied
- ? "all containers allocated"
+ ? ALL_CONTAINERS_ALLOCATED
: String.format("Awaiting %d containers", liveness.requestsOutstanding);
Hamlet.TABLE<DIV<Hamlet>> table1 = div.table();
table1.tr()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
new file mode 100644
index 0000000..6f21006
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.slider.client.SliderClient
+
+/**
+ * Overridde the test actions with a sleep command, so that developers
+ * can see what the application is up to
+ */
+class DemoAgentAAEcho extends TestAgentAAEcho {
+
+ @Override
+ protected void postLaunchActions(
+ SliderClient sliderClient,
+ String clustername,
+ String roleName,
+ Map<String, Integer> roles) {
+
+ def applicationReport = sliderClient.applicationReport
+ def url = applicationReport.trackingUrl
+ // spin repeating the URl in the logs so YARN chatter doesn't lose it
+ 1..5.each {
+ describe("Web UI is at $url")
+ sleep(60 *1000)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
index 80ff5a8..3835330 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -83,7 +83,7 @@ class TestAgentAAEcho extends TestAgentEcho {
*/
protected Map<String, Integer> buildRoleMap(String roleName) {
[
- (roleName): 2,
+ (roleName): 3,
];
}
@@ -108,12 +108,13 @@ class TestAgentAAEcho extends TestAgentEcho {
// flex size
// while running, ask for many more, expect them to still be outstanding
sleep(5000)
+ sliderClient.flex(clustername, [(roleName): 50]);
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
- sliderClient.flex(clustername, onlyOneEcho);
- // while running, flex it with no changes
- sliderClient.flex(clustername, [(roleName): 3]);
+ // while running, flex it to size = 1
sleep(1000)
+ sliderClient.flex(clustername, onlyOneEcho);
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
new file mode 100644
index 0000000..7b3a65d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
+import org.apache.slider.server.appmaster.state.RoleStatus
+
+/**
+ * class for basis of Anti-affine placement tests; sets up role2
+ * for anti-affinity
+ */
+@CompileStatic
+@Slf4j
+class BaseMockAppStateAATest extends BaseMockAppStateTest
+ implements MockRoles {
+
+ /** Role status for the base AA role */
+ RoleStatus aaRole
+
+ /** Role status for the AA role requiring a node with the gpu label */
+ RoleStatus gpuRole
+
+ @Override
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.roles = [
+ MockFactory.PROVIDER_ROLE0,
+ MockFactory.AAROLE_1_GPU,
+ MockFactory.AAROLE_2,
+ ]
+ bindingInfo
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ aaRole = lookupRole(MockFactory.AAROLE_2.name)
+ gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 01ca2f1..749e4fc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -21,23 +21,15 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.providers.PlacementPolicy
-import org.apache.slider.providers.ProviderRole
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAssignment
-import org.apache.slider.server.appmaster.state.NodeMap
import org.apache.slider.server.appmaster.state.RoleInstance
-import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
@@ -45,41 +37,12 @@ import org.junit.Test
*/
@CompileStatic
@Slf4j
-class TestMockAppStateAAPlacement extends BaseMockAppStateTest
+class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
implements MockRoles {
- /**
- * Patch up a "role2" role to have anti-affinity set
- */
- public static final ProviderRole AAROLE = new ProviderRole(
- MockRoles.ROLE2,
- 2,
- PlacementPolicy.ANTI_AFFINITY_REQUIRED,
- 2,
- 2,
- null)
-
- RoleStatus aaRole
private int NODES = 3
@Override
- AppStateBindingInfo buildBindingInfo() {
- def bindingInfo = super.buildBindingInfo()
- bindingInfo.roles = [
- MockFactory.PROVIDER_ROLE0,
- MockFactory.PROVIDER_ROLE1,
- AAROLE,
- ]
- bindingInfo
- }
-
- @Override
- void setup() {
- super.setup()
- aaRole = lookupRole(AAROLE.name)
- }
-
- @Override
MockYarnEngine createYarnEngine() {
new MockYarnEngine(NODES, 8)
}
@@ -288,7 +251,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateTest
public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
describe("Change the cluster size where the cluster size changes during a test sequence.")
aaRole.desired = NODES + 1
- List<AbstractRMOperation> operations = appState.reviewRequestAndReleaseNodes()
+ appState.reviewRequestAndReleaseNodes()
assert aaRole.AARequestOutstanding
assert NODES == aaRole.pendingAntiAffineRequests
def outcome = addNewNode()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index 790a80e..f0fed95 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -22,17 +22,11 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.providers.PlacementPolicy
-import org.apache.slider.providers.ProviderRole
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo
-import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
@@ -40,45 +34,22 @@ import org.junit.Test
*/
@CompileStatic
@Slf4j
-class TestMockLabelledAAPlacement extends BaseMockAppStateTest
+class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
implements MockRoles {
- /**
- * Patch up a "role2" role to have anti-affinity set and the label of GPU
- */
- public static final ProviderRole AAROLE = new ProviderRole(
- MockRoles.ROLE2,
- 2,
- PlacementPolicy.ANTI_AFFINITY_REQUIRED,
- 2,
- 2,
- "gpu")
-
- RoleStatus aaRole
private int NODES = 3
private int GPU_NODES = 2
private String HOST0 = "00000000"
private String HOST1 = "00000001"
- @Override
- AppStateBindingInfo buildBindingInfo() {
- def bindingInfo = super.buildBindingInfo()
- bindingInfo.roles = [
- MockFactory.PROVIDER_ROLE0,
- MockFactory.PROVIDER_ROLE1,
- AAROLE,
- ]
- bindingInfo
- }
@Override
void setup() {
super.setup()
- aaRole = lookupRole(AAROLE.name)
// node 1 is GPU
- updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, "gpu"))
- updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, "gpu"))
+ updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
@@ -87,7 +58,7 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateTest
}
void assertAllContainersAA() {
- assertAllContainersAA(aaRole.key)
+ assertAllContainersAA(gpuRole.key)
}
/**
@@ -101,12 +72,12 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateTest
" expect the final request to be unsatisfied until the cluster changes size")
//more than expected
int size = GPU_NODES
- aaRole.desired = size + 1
+ gpuRole.desired = size + 1
List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
- assert aaRole.AARequestOutstanding
+ assert gpuRole.AARequestOutstanding
- assert aaRole.pendingAntiAffineRequests == size
+ assert gpuRole.pendingAntiAffineRequests == size
for (int i = 0; i < size; i++) {
def iter = "Iteration $i role = $aaRole"
describe iter
@@ -127,9 +98,9 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateTest
}
}
// expect an outstanding AA request to be unsatisfied
- assert aaRole.actual < aaRole.desired
- assert !aaRole.requested
- assert !aaRole.AARequestOutstanding
+ assert gpuRole.actual < gpuRole.desired
+ assert !gpuRole.requested
+ assert !gpuRole.AARequestOutstanding
List<Container> allocatedContainers = engine.execute(operations, [])
assert 0 == allocatedContainers.size()
// in a review now, no more requests can be generated, as there is no space for AA placements,
@@ -153,10 +124,10 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateTest
@Test
public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
describe("Change the cluster size where the cluster size changes during a test sequence.")
- aaRole.desired = GPU_NODES + 1
+ gpuRole.desired = GPU_NODES + 1
List<AbstractRMOperation> operations = appState.reviewRequestAndReleaseNodes()
- assert aaRole.AARequestOutstanding
- assert GPU_NODES == aaRole.pendingAntiAffineRequests
+ assert gpuRole.AARequestOutstanding
+ assert GPU_NODES == gpuRole.pendingAntiAffineRequests
def outcome = addNewNode()
assert outcome.clusterChanged
// one call to cancel
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index bbd64f1..4bbfbd8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -44,6 +44,9 @@ class MockFactory implements MockRoles {
Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
*/
+ /**
+ * basic role
+ */
public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
MockRoles.ROLE0,
0,
@@ -51,7 +54,9 @@ class MockFactory implements MockRoles {
2,
1,
ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
- // role 1 is strict. timeout should be irrelevant; same as failures
+ /**
+ * role 1 is strict. timeout should be irrelevant; same as failures
+ */
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
1,
@@ -59,7 +64,10 @@ class MockFactory implements MockRoles {
2,
1,
ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
- // role 2: longer delay
+
+ /**
+ * role 2: longer delay
+ */
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
2,
@@ -68,6 +76,28 @@ class MockFactory implements MockRoles {
2,
ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+ /**
+ * Patch up a "role2" role to have anti-affinity set
+ */
+ public static final ProviderRole AAROLE_2 = new ProviderRole(
+ MockRoles.ROLE2,
+ 2,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 2,
+ 2,
+ null)
+
+ /**
+ * Patch up a "role1" role to have anti-affinity set and GPI as the label
+ */
+ public static final ProviderRole AAROLE_1_GPU = new ProviderRole(
+ MockRoles.ROLE1,
+ 1,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 2,
+ 1,
+ MockRoles.LABEL_GPU)
+
int appIdCount;
int attemptIdCount;
int containerIdCount;
@@ -83,7 +113,7 @@ class MockFactory implements MockRoles {
PROVIDER_ROLE1,
PROVIDER_ROLE2,
]
-
+
public static final int ROLE_COUNT = ROLES.size();
MockContainerId newContainerId() {
@@ -211,6 +241,5 @@ class MockFactory implements MockRoles {
MockContainerStatus newContainerStatus() {
return new MockContainerStatus()
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
index b44482d..5ee8f24 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
@@ -24,4 +24,6 @@ public interface MockRoles {
String ROLE1 = "role1"
String ROLE2 = "role2"
int ROLE_COUNT = 3
+ String LABEL_GPU = "gpu"
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
index c2ea837..a4db705 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -24,8 +24,10 @@ import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest
import org.apache.slider.server.appmaster.model.mock.*
import org.apache.slider.server.appmaster.state.ContainerOutcome
+import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
import org.apache.slider.server.appmaster.web.WebAppApiImpl
@@ -34,7 +36,7 @@ import org.junit.Test
@Slf4j
//@CompileStatic
-public class TestIndexBlock extends BaseMockAppStateTest {
+public class TestIndexBlock extends BaseMockAppStateAATest {
private IndexBlock indexBlock;
@@ -81,23 +83,51 @@ public class TestIndexBlock extends BaseMockAppStateTest {
public void testIndex() {
def role0 = role0Status
def role1 = role1Status
- role0.desired = 8
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role1.incRequested()
- role1.incRequested()
- role1.incRequested()
+ def role2 = role2Status
+
+ def role0_desired = 8
+
+ role0.desired = role0_desired
+ int role0_actual = 5
+ int role0_requested = role0_desired - role0_actual
+ role0_actual.times {
+ role0.incActual()
+ }
+ assert role0.getActual() == role0_actual
+ role0_requested.times {
+ role0.incRequested()
+ }
+ assert role0.getRequested() == role0_requested
+
+ def role0_failures = 2
+
role0.noteFailed(false, "", ContainerOutcome.Failed)
role0.noteFailed(true, "", ContainerOutcome.Failed)
+ // all aa roles fields are in the
+ def aarole_desired = 200
+ aaRole.desired = aarole_desired
+ def aarole_actual = 90
+ def aarole_active = 1
+ def aarole_requested = aarole_desired - aarole_actual
+ def aarole_pending = aarole_requested - 1
+ def aarole_failures = 0
+ aarole_actual.times {
+ aaRole.incActual()
+ }
+ assert aaRole.actual == aarole_actual
+ aaRole.outstandingAArequest = new OutstandingRequest(2, "")
+ // add a requested
+ aaRole.incRequested()
+ aaRole.pendingAntiAffineRequests = aarole_pending
+ assert aaRole.pendingAntiAffineRequests == aarole_pending
+
+ assert aaRole.actualAndRequested == aarole_desired
StringWriter sw = new StringWriter(64);
PrintWriter pw = new PrintWriter(sw);
Hamlet hamlet = new Hamlet(pw, 0, false);
-
+
int level = hamlet.nestLevel();
indexBlock.doIndex(hamlet, "accumulo");
@@ -106,17 +136,29 @@ public class TestIndexBlock extends BaseMockAppStateTest {
assertEquals(body, level, hamlet.nestLevel())
// verify role data came out
assert body.contains("role0")
- //
- assert body.contains("8")
- assert body.contains("5")
- assert body.contains("3")
- assert body.contains("2")
- assert body.contains("1")
-
+ assertContains(role0_desired, body)
+ assertContains(role0_actual, body)
+ assertContains(role0_requested, body)
+ assertContains(role0_failures, body)
+
assert body.contains("role1")
assert body.contains("role2")
+
+ assertContains(aarole_desired, body)
+ assertContains(aarole_actual, body)
+ assertContains(aarole_requested, body)
+ assertContains(aarole_failures, body)
+
// verify that the sorting took place
assert body.indexOf("role0") < body.indexOf("role1")
assert body.indexOf("role1") < body.indexOf("role2")
+
+ assert !body.contains(IndexBlock.ALL_CONTAINERS_ALLOCATED)
+ // role
+ }
+
+ def assertContains(int ex, String html) {
+ assertStringContains(Integer.toString(ex), html)
+
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy b/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
new file mode 100644
index 0000000..451bdae
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.management
+
+import org.apache.slider.server.appmaster.management.LongGauge
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+class TestGauges extends SliderTestBase {
+
+ @Test
+ public void testLongGaugeOperations() throws Throwable {
+ LongGauge gauge = new LongGauge();
+ assert gauge.get() == 0
+ gauge.inc()
+ assert gauge.get() == 1
+ gauge.inc()
+ assert gauge.get() == 2
+ gauge.inc()
+ assert gauge.get() == 3
+ assert gauge.getValue() == gauge.get()
+ assert gauge.count == gauge.get()
+
+ gauge.dec()
+ assert gauge.get() == 2
+ assert gauge.decToFloor(1) == 1
+ assert gauge.get() == 1
+ assert gauge.decToFloor(1) == 0
+ assert gauge.decToFloor(1) == 0
+ assert gauge.decToFloor(0) == 0
+
+ gauge.set(4)
+ assert gauge.decToFloor(8) == 0
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8efc4c2c/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index e1f2f75..ab81c46 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -118,11 +118,20 @@ class SliderTestUtils extends Assert {
JsonOutput.prettyPrint(JsonOutput.toJson(src))
}
+ /**
+ * Skip the test with a message
+ * @param message message logged and thrown
+ */
public static void skip(String message) {
log.warn("Skipping test: {}", message)
Assume.assumeTrue(message, false);
}
+ /**
+ * Skip the test with a message if condition holds
+ * @param condition predicate
+ * @param message message logged and thrown
+ */
public static void assume(boolean condition, String message) {
if (!condition) {
skip(message)
@@ -132,18 +141,25 @@ class SliderTestUtils extends Assert {
/**
* Skip a test if not running on Windows
*/
- public assumeWindows() {
+ public static void assumeWindows() {
assume(Shell.WINDOWS, "not windows")
}
/**
* Skip a test if running on Windows
*/
- public assumeNotWindows() {
+ public static void assumeNotWindows() {
assume(!Shell.WINDOWS, "windows")
}
/**
+ * skip a test on windows
+ */
+ public static void skipOnWindows() {
+ assumeNotWindows();
+ }
+
+ /**
* Equality size for a list
* @param left
* @param right
@@ -243,15 +259,6 @@ class SliderTestUtils extends Assert {
}
/**
- * skip a test on windows
- */
- public static void skipOnWindows() {
- if (Shell.WINDOWS) {
- skip("Not supported on windows")
- }
- }
-
- /**
* Assert that any needed libraries being present. On Unix none are needed;
* on windows they must be present
*/
@@ -1474,6 +1481,13 @@ class SliderTestUtils extends Assert {
}
}
+ /**
+ * Probe for a metric gauge holding a value.
+ *
+ * Keys: "url:String", "gauge:String", "desiredValue:int"
+ * @param args argument map
+ * @return success on the desired value, retry if not; fail on IOE
+ */
Outcome probeMetricGaugeValue(Map args) {
String url = requiredMapValue(args, "url")
String gauge = requiredMapValue(args, "gauge")
@@ -1490,4 +1504,13 @@ class SliderTestUtils extends Assert {
}
}
+ public static void assertStringContains(String expected, String text) {
+ assertNotNull("null text", text)
+ if (!text.contains(expected)) {
+ def message = "id not find $expected in \"$text\""
+ log.error(message)
+ fail(message)
+ }
+
+ }
}
[4/8] incubator-slider git commit: SLIDER-979 AM web UI to show state
of AA request
Posted by st...@apache.org.
SLIDER-979 AM web UI to show state of AA request
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/591ba99e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/591ba99e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/591ba99e
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 591ba99ecbcec2edc208cafec6b36cc84c6f23cf
Parents: 8efc4c2
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 16 20:07:59 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 16 20:07:59 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/proto/Messages.java | 191 ++++++++++++++-----
.../slider/api/proto/RestTypeMarshalling.java | 8 +-
.../types/ApplicationLivenessInformation.java | 6 +
.../slider/api/types/ComponentInformation.java | 4 +-
.../apache/slider/api/types/RoleStatistics.java | 66 +++++++
.../rest/SliderApplicationApiRestClient.java | 14 ++
.../appmaster/actions/ActionKillContainer.java | 2 +-
.../server/appmaster/actions/QueueService.java | 8 +-
.../slider/server/appmaster/state/AppState.java | 33 ++--
.../appmaster/state/ProviderAppState.java | 5 +
.../server/appmaster/state/RoleStatus.java | 20 ++
.../state/StateAccessForProviders.java | 7 +
.../web/rest/AbstractSliderResource.java | 7 +-
.../server/appmaster/web/rest/RestPaths.java | 11 ++
.../web/view/ClusterSpecificationBlock.java | 13 +-
.../appmaster/web/view/ContainerStatsBlock.java | 9 +-
.../server/appmaster/web/view/IndexBlock.java | 84 ++++----
.../server/appmaster/web/view/NavBlock.java | 26 ++-
.../appmaster/web/view/SliderHamletBlock.java | 56 ++++++
.../src/main/proto/SliderClusterMessages.proto | 1 +
.../agent/AgentMiniClusterTestBase.groovy | 7 +-
.../slider/agent/rest/TestStandaloneREST.groovy | 5 +-
.../slider/providers/agent/AgentTestBase.groovy | 1 +
.../providers/agent/DemoAgentAAEcho.groovy | 8 +-
.../providers/agent/TestAgentAAEcho.groovy | 29 ++-
.../slider/providers/agent/TestAgentEcho.groovy | 6 +-
.../org/apache/slider/test/KeysForTests.groovy | 3 +-
slider-core/src/test/python/agent.py | 25 ++-
slider-core/src/test/python/agent/main.py | 18 +-
slider-core/src/test/python/echo.py | 23 ++-
slider-core/src/test/python/metainfo.xml | 2 +-
31 files changed, 529 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
index 569660d..ed056f5 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -15110,6 +15110,16 @@ public final class Messages {
* <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
*/
int getPendingAntiAffineRequestCount();
+
+ // optional bool isAARequestOutstanding = 19;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ boolean hasIsAARequestOutstanding();
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ boolean getIsAARequestOutstanding();
}
/**
* Protobuf type {@code org.apache.slider.api.ComponentInformationProto}
@@ -15260,6 +15270,11 @@ public final class Messages {
pendingAntiAffineRequestCount_ = input.readInt32();
break;
}
+ case 152: {
+ bitField0_ |= 0x00020000;
+ isAARequestOutstanding_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -15659,6 +15674,22 @@ public final class Messages {
return pendingAntiAffineRequestCount_;
}
+ // optional bool isAARequestOutstanding = 19;
+ public static final int ISAAREQUESTOUTSTANDING_FIELD_NUMBER = 19;
+ private boolean isAARequestOutstanding_;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean hasIsAARequestOutstanding() {
+ return ((bitField0_ & 0x00020000) == 0x00020000);
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean getIsAARequestOutstanding() {
+ return isAARequestOutstanding_;
+ }
+
private void initFields() {
name_ = "";
priority_ = 0;
@@ -15678,6 +15709,7 @@ public final class Messages {
nodeFailed_ = 0;
preempted_ = 0;
pendingAntiAffineRequestCount_ = 0;
+ isAARequestOutstanding_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -15745,6 +15777,9 @@ public final class Messages {
if (((bitField0_ & 0x00010000) == 0x00010000)) {
output.writeInt32(18, pendingAntiAffineRequestCount_);
}
+ if (((bitField0_ & 0x00020000) == 0x00020000)) {
+ output.writeBool(19, isAARequestOutstanding_);
+ }
getUnknownFields().writeTo(output);
}
@@ -15831,6 +15866,10 @@ public final class Messages {
size += com.google.protobuf.CodedOutputStream
.computeInt32Size(18, pendingAntiAffineRequestCount_);
}
+ if (((bitField0_ & 0x00020000) == 0x00020000)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(19, isAARequestOutstanding_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -15941,6 +15980,11 @@ public final class Messages {
result = result && (getPendingAntiAffineRequestCount()
== other.getPendingAntiAffineRequestCount());
}
+ result = result && (hasIsAARequestOutstanding() == other.hasIsAARequestOutstanding());
+ if (hasIsAARequestOutstanding()) {
+ result = result && (getIsAARequestOutstanding()
+ == other.getIsAARequestOutstanding());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -16026,6 +16070,10 @@ public final class Messages {
hash = (37 * hash) + PENDINGANTIAFFINEREQUESTCOUNT_FIELD_NUMBER;
hash = (53 * hash) + getPendingAntiAffineRequestCount();
}
+ if (hasIsAARequestOutstanding()) {
+ hash = (37 * hash) + ISAAREQUESTOUTSTANDING_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getIsAARequestOutstanding());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -16176,6 +16224,8 @@ public final class Messages {
bitField0_ = (bitField0_ & ~0x00010000);
pendingAntiAffineRequestCount_ = 0;
bitField0_ = (bitField0_ & ~0x00020000);
+ isAARequestOutstanding_ = false;
+ bitField0_ = (bitField0_ & ~0x00040000);
return this;
}
@@ -16278,6 +16328,10 @@ public final class Messages {
to_bitField0_ |= 0x00010000;
}
result.pendingAntiAffineRequestCount_ = pendingAntiAffineRequestCount_;
+ if (((from_bitField0_ & 0x00040000) == 0x00040000)) {
+ to_bitField0_ |= 0x00020000;
+ }
+ result.isAARequestOutstanding_ = isAARequestOutstanding_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -16359,6 +16413,9 @@ public final class Messages {
if (other.hasPendingAntiAffineRequestCount()) {
setPendingAntiAffineRequestCount(other.getPendingAntiAffineRequestCount());
}
+ if (other.hasIsAARequestOutstanding()) {
+ setIsAARequestOutstanding(other.getIsAARequestOutstanding());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -17122,6 +17179,39 @@ public final class Messages {
return this;
}
+ // optional bool isAARequestOutstanding = 19;
+ private boolean isAARequestOutstanding_ ;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean hasIsAARequestOutstanding() {
+ return ((bitField0_ & 0x00040000) == 0x00040000);
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean getIsAARequestOutstanding() {
+ return isAARequestOutstanding_;
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public Builder setIsAARequestOutstanding(boolean value) {
+ bitField0_ |= 0x00040000;
+ isAARequestOutstanding_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public Builder clearIsAARequestOutstanding() {
+ bitField0_ = (bitField0_ & ~0x00040000);
+ isAARequestOutstanding_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:org.apache.slider.api.ComponentInformationProto)
}
@@ -34023,7 +34113,7 @@ public final class Messages {
" \002(\t\022\023\n\013application\030\003 \002(\t\"`\n#Application" +
"LivenessInformationProto\022\034\n\024allRequestsS" +
"atisfied\030\001 \001(\010\022\033\n\023requestsOutstanding\030\002 " +
- "\001(\005\"\216\003\n\031ComponentInformationProto\022\014\n\004nam",
+ "\001(\005\"\256\003\n\031ComponentInformationProto\022\014\n\004nam",
"e\030\001 \001(\t\022\020\n\010priority\030\002 \001(\005\022\017\n\007desired\030\003 \001" +
"(\005\022\016\n\006actual\030\004 \001(\005\022\021\n\treleasing\030\005 \001(\005\022\021\n" +
"\trequested\030\006 \001(\005\022\016\n\006failed\030\007 \001(\005\022\017\n\007star" +
@@ -34033,54 +34123,55 @@ public final class Messages {
"(\005\022\022\n\ncontainers\030\016 \003(\t\022\026\n\016failedRecently" +
"\030\017 \001(\005\022\022\n\nnodeFailed\030\020 \001(\005\022\021\n\tpreempted\030" +
"\021 \001(\005\022%\n\035pendingAntiAffineRequestCount\030\022" +
- " \001(\005\"\210\002\n\031ContainerInformationProto\022\023\n\013co",
- "ntainerId\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010re" +
- "leased\030\003 \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030" +
- "\005 \001(\005\022\023\n\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime" +
- "\030\007 \001(\003\022\021\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(" +
- "\t\022\014\n\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplac" +
- "ement\030\014 \001(\t\022\022\n\nappVersion\030\r \001(\t\"N\n\024PingI" +
- "nformationProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 " +
- "\001(\t\022\014\n\004body\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\325\001\n\031Node" +
- "EntryInformationProto\022\020\n\010priority\030\001 \002(\005\022" +
- "\021\n\trequested\030\002 \002(\005\022\020\n\010starting\030\003 \002(\005\022\023\n\013",
- "startFailed\030\004 \002(\005\022\016\n\006failed\030\005 \002(\005\022\026\n\016fai" +
- "ledRecently\030\006 \002(\005\022\021\n\tpreempted\030\007 \002(\005\022\014\n\004" +
- "live\030\010 \002(\005\022\021\n\treleasing\030\t \002(\005\022\020\n\010lastUse" +
- "d\030\n \002(\003\"\334\001\n\024NodeInformationProto\022\020\n\010host" +
- "name\030\001 \002(\t\022\r\n\005state\030\002 \002(\t\022\023\n\013httpAddress" +
- "\030\003 \002(\t\022\020\n\010rackName\030\004 \002(\t\022\016\n\006labels\030\005 \002(\t" +
- "\022\024\n\014healthReport\030\006 \002(\t\022\023\n\013lastUpdated\030\007 " +
- "\002(\003\022A\n\007entries\030\010 \003(\01320.org.apache.slider" +
- ".api.NodeEntryInformationProto\"\026\n\024GetMod" +
- "elRequestProto\"\035\n\033GetModelDesiredRequest",
- "Proto\"$\n\"GetModelDesiredAppconfRequestPr" +
- "oto\"&\n$GetModelDesiredResourcesRequestPr" +
- "oto\"%\n#GetModelResolvedAppconfRequestPro" +
- "to\"\'\n%GetModelResolvedResourcesRequestPr" +
- "oto\"#\n!GetModelLiveResourcesRequestProto" +
- "\"\037\n\035GetLiveContainersRequestProto\"u\n\036Get" +
- "LiveContainersResponseProto\022\r\n\005names\030\001 \003" +
- "(\t\022D\n\ncontainers\030\002 \003(\01320.org.apache.slid" +
- "er.api.ContainerInformationProto\"3\n\034GetL" +
- "iveContainerRequestProto\022\023\n\013containerId\030",
- "\001 \002(\t\"\037\n\035GetLiveComponentsRequestProto\"u" +
- "\n\036GetLiveComponentsResponseProto\022\r\n\005name" +
- "s\030\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org.apache" +
- ".slider.api.ComponentInformationProto\",\n" +
- "\034GetLiveComponentRequestProto\022\014\n\004name\030\001 " +
- "\002(\t\"$\n\"GetApplicationLivenessRequestProt" +
- "o\"\023\n\021EmptyPayloadProto\" \n\020WrappedJsonPro" +
- "to\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificateStoreR" +
- "equestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013request" +
- "erId\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004type\030\004 \002",
- "(\t\"1\n GetCertificateStoreResponseProto\022\r" +
- "\n\005store\030\001 \002(\014\"\032\n\030GetLiveNodesRequestProt" +
- "o\"f\n\031GetLiveNodesResponseProto\022\r\n\005names\030" +
- "\001 \003(\t\022:\n\005nodes\030\002 \003(\0132+.org.apache.slider" +
- ".api.NodeInformationProto\"\'\n\027GetLiveNode" +
- "RequestProto\022\014\n\004name\030\001 \002(\tB-\n\033org.apache" +
- ".slider.api.protoB\010Messages\210\001\001\240\001\001"
+ " \001(\005\022\036\n\026isAARequestOutstanding\030\023 \001(\010\"\210\002\n",
+ "\031ContainerInformationProto\022\023\n\013containerI" +
+ "d\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010released\030\003" +
+ " \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030\005 \001(\005\022\023\n" +
+ "\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime\030\007 \001(\003\022\021" +
+ "\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(\t\022\014\n\004hos" +
+ "t\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplacement\030\014 " +
+ "\001(\t\022\022\n\nappVersion\030\r \001(\t\"N\n\024PingInformati" +
+ "onProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 \001(\t\022\014\n\004b" +
+ "ody\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\325\001\n\031NodeEntryInf" +
+ "ormationProto\022\020\n\010priority\030\001 \002(\005\022\021\n\treque",
+ "sted\030\002 \002(\005\022\020\n\010starting\030\003 \002(\005\022\023\n\013startFai" +
+ "led\030\004 \002(\005\022\016\n\006failed\030\005 \002(\005\022\026\n\016failedRecen" +
+ "tly\030\006 \002(\005\022\021\n\tpreempted\030\007 \002(\005\022\014\n\004live\030\010 \002" +
+ "(\005\022\021\n\treleasing\030\t \002(\005\022\020\n\010lastUsed\030\n \002(\003\"" +
+ "\334\001\n\024NodeInformationProto\022\020\n\010hostname\030\001 \002" +
+ "(\t\022\r\n\005state\030\002 \002(\t\022\023\n\013httpAddress\030\003 \002(\t\022\020" +
+ "\n\010rackName\030\004 \002(\t\022\016\n\006labels\030\005 \002(\t\022\024\n\014heal" +
+ "thReport\030\006 \002(\t\022\023\n\013lastUpdated\030\007 \002(\003\022A\n\007e" +
+ "ntries\030\010 \003(\01320.org.apache.slider.api.Nod" +
+ "eEntryInformationProto\"\026\n\024GetModelReques",
+ "tProto\"\035\n\033GetModelDesiredRequestProto\"$\n" +
+ "\"GetModelDesiredAppconfRequestProto\"&\n$G" +
+ "etModelDesiredResourcesRequestProto\"%\n#G" +
+ "etModelResolvedAppconfRequestProto\"\'\n%Ge" +
+ "tModelResolvedResourcesRequestProto\"#\n!G" +
+ "etModelLiveResourcesRequestProto\"\037\n\035GetL" +
+ "iveContainersRequestProto\"u\n\036GetLiveCont" +
+ "ainersResponseProto\022\r\n\005names\030\001 \003(\t\022D\n\nco" +
+ "ntainers\030\002 \003(\01320.org.apache.slider.api.C" +
+ "ontainerInformationProto\"3\n\034GetLiveConta",
+ "inerRequestProto\022\023\n\013containerId\030\001 \002(\t\"\037\n" +
+ "\035GetLiveComponentsRequestProto\"u\n\036GetLiv" +
+ "eComponentsResponseProto\022\r\n\005names\030\001 \003(\t\022" +
+ "D\n\ncomponents\030\002 \003(\01320.org.apache.slider." +
+ "api.ComponentInformationProto\",\n\034GetLive" +
+ "ComponentRequestProto\022\014\n\004name\030\001 \002(\t\"$\n\"G" +
+ "etApplicationLivenessRequestProto\"\023\n\021Emp" +
+ "tyPayloadProto\" \n\020WrappedJsonProto\022\014\n\004js" +
+ "on\030\001 \002(\t\"h\n\037GetCertificateStoreRequestPr" +
+ "oto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013requesterId\030\002 \002",
+ "(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004type\030\004 \002(\t\"1\n Ge" +
+ "tCertificateStoreResponseProto\022\r\n\005store\030" +
+ "\001 \002(\014\"\032\n\030GetLiveNodesRequestProto\"f\n\031Get" +
+ "LiveNodesResponseProto\022\r\n\005names\030\001 \003(\t\022:\n" +
+ "\005nodes\030\002 \003(\0132+.org.apache.slider.api.Nod" +
+ "eInformationProto\"\'\n\027GetLiveNodeRequestP" +
+ "roto\022\014\n\004name\030\001 \002(\tB-\n\033org.apache.slider." +
+ "api.protoB\010Messages\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -34236,7 +34327,7 @@ public final class Messages {
internal_static_org_apache_slider_api_ComponentInformationProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_ComponentInformationProto_descriptor,
- new java.lang.String[] { "Name", "Priority", "Desired", "Actual", "Releasing", "Requested", "Failed", "Started", "StartFailed", "Completed", "TotalRequested", "FailureMessage", "PlacementPolicy", "Containers", "FailedRecently", "NodeFailed", "Preempted", "PendingAntiAffineRequestCount", });
+ new java.lang.String[] { "Name", "Priority", "Desired", "Actual", "Releasing", "Requested", "Failed", "Started", "StartFailed", "Completed", "TotalRequested", "FailureMessage", "PlacementPolicy", "Containers", "FailedRecently", "NodeFailed", "Preempted", "PendingAntiAffineRequestCount", "IsAARequestOutstanding", });
internal_static_org_apache_slider_api_ContainerInformationProto_descriptor =
getDescriptor().getMessageTypes().get(25);
internal_static_org_apache_slider_api_ContainerInformationProto_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 85a8358..8dcf65f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -86,7 +86,12 @@ public class RestTypeMarshalling {
if (wire.hasFailureMessage()) {
info.failureMessage = wire.getFailureMessage();
}
- info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ if (wire.hasPendingAntiAffineRequestCount()) {
+ info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ }
+ if (wire.hasIsAARequestOutstanding()) {
+ info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+ }
return info;
}
@@ -137,6 +142,7 @@ public class RestTypeMarshalling {
builder.addAllContainers(info.containers);
}
builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
+ builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
index 4dd2cb7..9879d05 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
@@ -30,9 +30,15 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ApplicationLivenessInformation {
+ /** flag set if the cluster is at size */
public boolean allRequestsSatisfied;
+
+ /** number of outstanding requests: those needed to satisfy */
public int requestsOutstanding;
+ /** number of requests submitted to YARN */
+ public int activeRequests;
+
@Override
public String toString() {
final StringBuilder sb =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index 9d8a4ee..3771988 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -53,6 +53,7 @@ public class ComponentInformation {
public int failed, started, startFailed, completed, totalRequested;
public int nodeFailed, failedRecently, preempted;
public int pendingAntiAffineRequestCount;
+ public boolean isAARequestOutstanding;
public String failureMessage;
public List<String> containers;
@@ -84,12 +85,13 @@ public class ComponentInformation {
new StringBuilder("ComponentInformation{");
sb.append(", name='").append(name).append('\'');
sb.append(", actual=").append(actual);
- sb.append(", anti-affine pending").append(pendingAntiAffineRequestCount);
sb.append(", completed=").append(completed);
sb.append(", desired=").append(desired);
sb.append(", failed=").append(failed);
sb.append(", failureMessage='").append(failureMessage).append('\'');
sb.append(", placementPolicy=").append(placementPolicy);
+ sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding);
+ sb.append(", pendingAntiAffineRequestCount").append(pendingAntiAffineRequestCount);
sb.append(", priority=").append(priority);
sb.append(", releasing=").append(releasing);
sb.append(", requested=").append(requested);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
new file mode 100644
index 0000000..c926600
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+/**
+ * Simple role statistics for state views; can be generated by RoleStatus
+ * instances, and aggregated for summary information.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RoleStatistics {
+ public long activeAA = 0L;
+ public long actual = 0L;
+ public long completed = 0L;
+ public long desired = 0L;
+ public long failed = 0L;
+ public long failedRecently = 0L;
+ public long limitsExceeded = 0L;
+ public long nodeFailed = 0L;
+ public long preempted = 0L;
+ public long releasing = 0L;
+ public long requested = 0L;
+ public long started = 0L;
+ public long startFailed = 0L;
+ public long totalRequested = 0L;
+
+ /**
+ * Add another statistics instance
+ * @param that the other value
+ * @return this entry
+ */
+ public RoleStatistics add(final RoleStatistics that) {
+ activeAA += that.activeAA;
+ actual += that.actual;
+ completed += that.completed;
+ desired += that.desired;
+ failed += that.failed;
+ failedRecently += that.failedRecently;
+ limitsExceeded += that.limitsExceeded;
+ nodeFailed += that.nodeFailed;
+ preempted += that.preempted;
+ releasing += that.releasing;
+ requested += that.requested;
+ started += that.started;
+ startFailed += that.totalRequested;
+ totalRequested += that.totalRequested;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
index ce2817a..54c60d1 100644
--- a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
@@ -25,6 +25,7 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.filter.LoggingFilter;
import com.sun.jersey.api.representation.Form;
import org.apache.commons.lang.StringUtils;
import org.apache.slider.api.types.ApplicationLivenessInformation;
@@ -68,6 +69,19 @@ public class SliderApplicationApiRestClient extends BaseRestClient
this.appResource = appResource;
}
+ /**
+ * Create an instance
+ * @param jerseyClient jersey client for operations
+ * @param appmaster URL of appmaster/proxy to AM
+ */
+ public SliderApplicationApiRestClient(Client jerseyClient, String appmaster) {
+ super(jerseyClient);
+ WebResource amResource = jerseyClient.resource(appmaster);
+ amResource.type(MediaType.APPLICATION_JSON);
+ this.appResource = amResource.path(SLIDER_PATH_APPLICATION);
+ }
+
+
@Override
public String toString() {
final StringBuilder sb =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 1aa9088..7446e82 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -77,7 +77,7 @@ public class ActionKillContainer extends AsyncAction {
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
- List<AbstractRMOperation> opsList = new LinkedList<AbstractRMOperation>();
+ List<AbstractRMOperation> opsList = new LinkedList<>();
ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
opsList.add(release);
//now apply the operations
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 146dea4..34acade 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -58,20 +58,20 @@ implements Runnable, QueueAccess {
* Immediate actions.
*/
public final BlockingDeque<AsyncAction> actionQueue =
- new LinkedBlockingDeque<AsyncAction>();
+ new LinkedBlockingDeque<>();
/**
* Actions to be scheduled in the future
*/
- public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<AsyncAction>();
+ public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
/**
* Map of renewing actions by name ... this is to allow them to
* be cancelled by name
*/
private final Map<String, RenewingAction<? extends AsyncAction>> renewingActions
- = new ConcurrentHashMap<String, RenewingAction<? extends AsyncAction>>();
-
+ = new ConcurrentHashMap<>();
+
/**
* Create a queue instance with a single thread executor
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 4152a89..2da5d36 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -46,6 +46,7 @@ import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -250,12 +251,6 @@ public class AppState {
private final LongGauge outstandingContainerRequests = new LongGauge();
/**
- * Track the number of pending (not yet active) requests
- * Important: this does not include AA requests which are yet to be issued.
- */
- private final LongGauge pendingAARequests = new LongGauge();
-
- /**
* Map of requested nodes. This records the command used to start it,
* resources, etc. When container started callback is received,
* the node is promoted from here to the containerMap
@@ -382,10 +377,6 @@ public class AppState {
startFailedContainerCount.inc();
}
- public long getTotalOutstandingRequests() {
- return outstandingContainerRequests.get() +
- pendingAARequests.get();
- }
public AtomicInteger getCompletionOfNodeNotInLiveListEvent() {
return completionOfNodeNotInLiveListEvent;
}
@@ -1777,9 +1768,11 @@ public class AppState {
*/
public ApplicationLivenessInformation getApplicationLivenessInformation() {
ApplicationLivenessInformation li = new ApplicationLivenessInformation();
- int outstanding = outstandingContainerRequests.intValue();
+ RoleStatistics stats = getRoleStatistics();
+ int outstanding = (int)(stats.desired - stats.actual);
li.requestsOutstanding = outstanding;
li.allRequestsSatisfied = outstanding <= 0;
+ li.activeRequests = (int)stats.requested;
return li;
}
@@ -1808,6 +1801,18 @@ public class AppState {
}
/**
+ * Get the aggregate statistics across all roles
+ * @return role statistics
+ */
+ public RoleStatistics getRoleStatistics() {
+ RoleStatistics stats = new RoleStatistics();
+ for (RoleStatus role : getRoleStatusMap().values()) {
+ stats.add(role.getStatistics());
+ }
+ return stats;
+ }
+
+ /**
* Get a snapshot of component information.
* <p>
* This does <i>not</i> include any container list, which
@@ -1817,8 +1822,7 @@ public class AppState {
public Map<String, ComponentInformation> getComponentInfoSnapshot() {
Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
- Map<String, ComponentInformation> results =
- new HashMap<String, ComponentInformation>(
+ Map<String, ComponentInformation> results = new HashMap<>(
statusMap.size());
for (RoleStatus status : statusMap.values()) {
@@ -2372,9 +2376,10 @@ public class AppState {
sb.append(", startFailedContainerCount=").append(startFailedContainerCount);
sb.append(", surplusContainers=").append(surplusContainers);
sb.append(", failedContainerCount=").append(failedContainerCount);
- sb.append(", outstandingContainerRequests=")
+ sb.append(", outstanding non-AA Container Requests=")
.append(outstandingContainerRequests);
sb.append('}');
return sb.toString();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 82b2f2a..1d96a1c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -26,6 +26,7 @@ import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -298,4 +299,8 @@ public class ProviderAppState implements StateAccessForProviders {
return appState.getRoleHistory().getNodeInformation(hostname);
}
+ @Override
+ public RoleStatistics getRoleStatistics() {
+ return appState.getRoleStatistics();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index b530d18..0fc3dc2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -21,6 +21,7 @@ package org.apache.slider.server.appmaster.state;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.LongGauge;
@@ -441,6 +442,7 @@ public final class RoleStatus implements Cloneable {
info.nodeFailed = nodeFailed.intValue();
info.preempted = preempted.intValue();
info.pendingAntiAffineRequestCount = pendingAntiAffineRequests.intValue();
+ info.isAARequestOutstanding = isAARequestOutstanding();
return info;
}
@@ -494,4 +496,22 @@ public final class RoleStatus implements Cloneable {
resource.setVirtualCores(resourceRequirements.getVirtualCores());
return resource;
}
+
+ public synchronized RoleStatistics getStatistics() {
+ RoleStatistics stats = new RoleStatistics();
+ stats.activeAA = isAARequestOutstanding() ? 1: 0;
+ stats.actual = actual.get();
+ stats.desired = desired.get();
+ stats.failed = failed.get();
+ stats.limitsExceeded = limitsExceeded.get();
+ stats.nodeFailed = nodeFailed.get();
+ stats.preempted = preempted.get();
+ stats.releasing = releasing.get();
+ stats.requested = requested.get();
+ stats.started = started.get();
+ stats.startFailed = startFailed.get();
+ stats.totalRequested = totalRequested.get();
+ return stats;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index 2fc00b2..ad91183 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -27,6 +27,7 @@ import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -303,4 +304,10 @@ public interface StateAccessForProviders {
* @return the information, or null if there is no information held.
*/
NodeInformation getNodeInformation(String hostname);
+
+ /**
+ * Get the aggregate statistics across all roles
+ * @return role statistics
+ */
+ RoleStatistics getRoleStatistics();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index cfddb12..7ff83b6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -45,12 +45,11 @@ public abstract class AbstractSliderResource {
protected final WebAppApi slider;
protected final MetricsAndMonitoring metricsAndMonitoring;
- public AbstractSliderResource(WebAppApi slider) {
+ protected AbstractSliderResource(WebAppApi slider) {
this.slider = slider;
metricsAndMonitoring = slider.getMetricsAndMonitoring();
}
-
/**
* Generate a redirect to the WASL
* @param request to base the URL on
@@ -105,6 +104,7 @@ public abstract class AbstractSliderResource {
protected void mark(String verb, String path) {
metricsAndMonitoring.markMeterAndCounter(verb + "-" + path);
}
+
/**
* Mark an GET operation on a path
* @param verb HTTP Verb
@@ -129,6 +129,7 @@ public abstract class AbstractSliderResource {
protected void markGet(String path, String subpath) {
mark("GET", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
@@ -136,6 +137,7 @@ public abstract class AbstractSliderResource {
protected void markPost(String path, String subpath) {
mark("POST", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
@@ -143,6 +145,7 @@ public abstract class AbstractSliderResource {
protected void markPut(String path, String subpath) {
mark("PUT", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 424107c..b90eb62 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -148,4 +148,15 @@ public class RestPaths {
public static final String ACTION = "/action";
public static final String ACTION_PING = ACTION + "/ping";
public static final String ACTION_STOP = ACTION + "/stop";
+
+ /**
+ * Path to a role
+ * @param name role name
+ * @return a path to it
+ */
+ public String pathToRole(String name) {
+
+ // ws/v1/slider/application/live/components/$name
+ return SLIDER_PATH_APPLICATION + LIVE_COMPONENTS + "/" + name;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
index 137c476..2f02f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
@@ -18,23 +18,16 @@ package org.apache.slider.server.appmaster.web.view;
import com.google.inject.Inject;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
*
*/
-public class ClusterSpecificationBlock extends HtmlBlock {
- private static final Logger log = LoggerFactory.getLogger(ClusterSpecificationBlock.class);
-
- private StateAccessForProviders appState;
+public class ClusterSpecificationBlock extends SliderHamletBlock {
@Inject
public ClusterSpecificationBlock(WebAppApi slider) {
- this.appState = slider.getAppState();
+ super(slider);
}
@Override
@@ -50,7 +43,7 @@ public class ClusterSpecificationBlock extends HtmlBlock {
pre().
_(getJson())._()._();
}
-
+
/**
* Get the JSON, catching any exceptions and returning error text instead
* @return
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
index 65d8b39..56285c2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -26,12 +26,10 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import java.io.Serializable;
@@ -45,7 +43,7 @@ import java.util.Map.Entry;
/**
*
*/
-public class ContainerStatsBlock extends HtmlBlock {
+public class ContainerStatsBlock extends SliderHamletBlock {
private static final String EVEN = "even", ODD = "odd", BOLD = "bold", SCHEME = "http://", PATH = "/node/container/";
@@ -54,11 +52,9 @@ public class ContainerStatsBlock extends HtmlBlock {
protected static final Function<Entry<String,Long>,Entry<TableContent,Long>> stringLongPairFunc = toTableContentFunction();
protected static final Function<Entry<String,String>,Entry<TableContent,String>> stringStringPairFunc = toTableContentFunction();
- private WebAppApi slider;
-
@Inject
public ContainerStatsBlock(WebAppApi slider) {
- this.slider = slider;
+ super(slider);
}
/**
@@ -93,7 +89,6 @@ public class ContainerStatsBlock extends HtmlBlock {
@Override
protected void render(Block html) {
- StateAccessForProviders appState = slider.getAppState();
final Map<String,RoleInstance> containerInstances = getContainerInstances(
appState.cloneOwnedContainerList());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 41e1c01..8152f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -21,28 +21,28 @@ import com.google.inject.Inject;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
+
/**
*
*/
-public class IndexBlock extends HtmlBlock {
+public class IndexBlock extends SliderHamletBlock {
private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
/**
@@ -52,13 +52,9 @@ public class IndexBlock extends HtmlBlock {
*/
public static final String ALL_CONTAINERS_ALLOCATED = "all containers allocated";
- private StateAccessForProviders appView;
- private ProviderService providerService;
-
@Inject
public IndexBlock(WebAppApi slider) {
- this.appView = slider.getAppState();
- this.providerService = slider.getProviderService();
+ super(slider);
}
@Override
@@ -71,7 +67,8 @@ public class IndexBlock extends HtmlBlock {
// An extra method to make testing easier since you can't make an instance of Block
@VisibleForTesting
protected void doIndex(Hamlet html, String providerName) {
- ClusterDescription clusterStatus = appView.getClusterStatus();
+ ClusterDescription clusterStatus = appState.getClusterStatus();
+ RoleStatistics roleStats = appState.getRoleStatistics();
String name = clusterStatus.name;
if (name != null && (name.startsWith(" ") || name.endsWith(" "))) {
name = "'" + name + "'";
@@ -81,9 +78,8 @@ public class IndexBlock extends HtmlBlock {
"Application: " + name);
ApplicationLivenessInformation liveness =
- appView.getApplicationLivenessInformation();
- String livestatus =
- liveness.allRequestsSatisfied
+ appState.getApplicationLivenessInformation();
+ String livestatus = liveness.allRequestsSatisfied
? ALL_CONTAINERS_ALLOCATED
: String.format("Awaiting %d containers", liveness.requestsOutstanding);
Hamlet.TABLE<DIV<Hamlet>> table1 = div.table();
@@ -93,7 +89,7 @@ public class IndexBlock extends HtmlBlock {
._();
table1.tr()
.td("Total number of containers")
- .td(Integer.toString(appView.getNumOwnedContainers()))
+ .td(Integer.toString(appState.getNumOwnedContainers()))
._();
table1.tr()
.td("Create time: ")
@@ -121,26 +117,40 @@ public class IndexBlock extends HtmlBlock {
html.div("container_instances").h3("Component Instances");
Hamlet.TABLE<DIV<Hamlet>> table = div.table();
- table.tr()
- .td("Component")
- .td("Desired")
- .td("Actual")
- .td("Outstanding Requests")
- .td("Failed")
- .td("Failed to start")
- ._();
-
- List<RoleStatus> roleStatuses = appView.cloneRoleStatusList();
+ Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> tr = table.thead().tr();
+ trb(tr, "Component");
+ trb(tr, "Desired");
+ trb(tr, "Actual");
+ trb(tr, "Outstanding Requests");
+ trb(tr, "Failed");
+ trb(tr, "Failed to start");
+ trb(tr, "Placement");
+ tr._()._();
+
+ List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
Collections.sort(roleStatuses, new RoleStatus.CompareByName());
for (RoleStatus status : roleStatuses) {
+ String roleName = status.getName();
+ String nameUrl = apiPath(LIVE_COMPONENTS) + "/" + roleName;
+ String aatext;
+ if (status.isAntiAffinePlacement()) {
+ int outstanding = status.isAARequestOutstanding() ? 1: 0;
+ int pending = (int)status.getPendingAntiAffineRequests();
+ aatext = String.format("Anti-affine: %d outstanding %s, %d pending %s",
+ outstanding, plural(outstanding, "request"),
+ pending, plural(pending, "request"));
+ } else {
+ aatext = "";
+ }
table.tr()
- .td(status.getName())
- .td(String.format("%d", status.getDesired()))
- .td(String.format("%d", status.getActual()))
- .td(String.format("%d", status.getRequested()))
- .td(String.format("%d", status.getFailed()))
- .td(String.format("%d", status.getStartFailed()))
- ._();
+ .td().a(nameUrl, roleName)._()
+ .td(String.format("%d", status.getDesired()))
+ .td(String.format("%d", status.getActual()))
+ .td(String.format("%d", status.getRequested()))
+ .td(String.format("%d", status.getFailed()))
+ .td(String.format("%d", status.getStartFailed()))
+ .td(aatext)
+ ._();
}
table._()._();
@@ -155,12 +165,21 @@ public class IndexBlock extends HtmlBlock {
ul._()._();
}
+ private String plural(int n, String text) {
+ return n == 1 ? text : (text + "s");
+ }
+
+ private void trb(Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> tr,
+ String text) {
+ tr.td().b(text)._();
+ }
+
private String getProviderName() {
return providerService.getHumanName();
}
private String getInfoAvoidingNulls(String key) {
- String createTime = appView.getClusterStatus().getInfo(key);
+ String createTime = appState.getClusterStatus().getInfo(key);
return null == createTime ? "N/A" : createTime;
}
@@ -183,4 +202,5 @@ public class IndexBlock extends HtmlBlock {
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
index b2327ba..515b1a3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
@@ -16,16 +16,21 @@
*/
package org.apache.slider.server.appmaster.web.view;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import com.google.inject.Inject;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+
import static org.apache.slider.server.appmaster.web.SliderAMWebApp.*;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-
/**
*
*/
-public class NavBlock extends HtmlBlock {
+public class NavBlock extends SliderHamletBlock {
+
+ @Inject
+ public NavBlock(WebAppApi slider) {
+ super(slider);
+ }
@Override
protected void render(Block html) {
@@ -48,20 +53,11 @@ public class NavBlock extends HtmlBlock {
li().a(apiPath(LIVE_RESOURCES), "Resources")._().
li().a(apiPath(LIVE_COMPONENTS), "Components")._().
li().a(apiPath(LIVE_CONTAINERS), "Containers")._().
+ li().a(apiPath(LIVE_NODES), "Nodes")._().
+ li().a(apiPath(LIVE_STATISTICS), "Statistics")._().
li().a(apiPath(LIVE_LIVENESS), "Liveness")._()
._()
._();
}
- private String rootPath(String absolutePath) {
- return root_url(absolutePath);
- }
-
- private String relPath(String... args) {
- return ujoin(this.prefix(), args);
- }
- private String apiPath(String api) {
- return root_url(SLIDER_PATH_APPLICATION, api);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
new file mode 100644
index 0000000..82d7c8f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.view;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
+
+/**
+ * Anything we want to share across slider hamlet blocks
+ */
+public abstract class SliderHamletBlock extends HtmlBlock {
+
+ protected final StateAccessForProviders appState;
+ protected final ProviderService providerService;
+ protected final RestPaths restPaths = new RestPaths();
+
+ public SliderHamletBlock(WebAppApi slider) {
+ this.appState = slider.getAppState();
+ this.providerService = slider.getProviderService();
+ }
+
+ protected String rootPath(String absolutePath) {
+ return root_url(absolutePath);
+ }
+
+ protected String relPath(String... args) {
+ return ujoin(this.prefix(), args);
+ }
+
+ protected String apiPath(String api) {
+ return root_url(SLIDER_PATH_APPLICATION, api);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto
index 50c10e4..2836454 100644
--- a/slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/slider-core/src/main/proto/SliderClusterMessages.proto
@@ -258,6 +258,7 @@ message ComponentInformationProto {
optional int32 nodeFailed = 16;
optional int32 preempted = 17;
optional int32 pendingAntiAffineRequestCount = 18;
+ optional bool isAARequestOutstanding = 19;
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index a7bc0a3..41bb7b1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -27,15 +27,13 @@ import org.apache.commons.io.FileUtils
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
+import org.apache.slider.test.KeysForTests
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.AfterClass
import org.junit.BeforeClass
import org.junit.rules.TemporaryFolder
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
/**
* test base for agent clusters
@@ -43,8 +41,7 @@ import org.slf4j.LoggerFactory
@CompileStatic
@Slf4j
public abstract class AgentMiniClusterTestBase
-extends YarnZKMiniClusterTestBase {
- private static Logger LOG = LoggerFactory.getLogger(AgentMiniClusterTestBase)
+extends YarnZKMiniClusterTestBase implements KeysForTests {
protected static File agentConf
protected static File agentDef
protected static Map<String, String> agentDefOptions
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 7cb1837..0bd1df0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -32,6 +32,7 @@ import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.server.appmaster.rpc.RpcBinder
+import org.apache.slider.test.KeysForTests
import org.junit.Test
import static org.apache.slider.server.appmaster.management.MetricsKeys.*
@@ -39,7 +40,7 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.*
@CompileStatic
@Slf4j
-class TestStandaloneREST extends AgentMiniClusterTestBase {
+class TestStandaloneREST extends AgentMiniClusterTestBase {
@Test
public void testStandaloneREST() throws Throwable {
@@ -88,7 +89,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
// this should be from AM launch itself
awaitGaugeValue(
appendToURL(proxyAM, SYSTEM_METRICS_JSON),
- "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag",
+ NODES_UPDATED_FLAG_METRIC,
1,
WEB_STARTUP_TIME * 2, 500)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 0e5cd00..e5951ac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -136,6 +136,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
boolean create,
boolean blockUntilRunning) {
+ YarnConfiguration conf = testConfiguration
def clusterOps = [:]
return createOrBuildCluster(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
index 6f21006..8606417 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
@@ -18,7 +18,6 @@
package org.apache.slider.providers.agent
-import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.slider.client.SliderClient
/**
@@ -37,7 +36,12 @@ class DemoAgentAAEcho extends TestAgentAAEcho {
def applicationReport = sliderClient.applicationReport
def url = applicationReport.trackingUrl
// spin repeating the URl in the logs so YARN chatter doesn't lose it
- 1..5.each {
+ describe("Web UI is at $url")
+
+ // run the superclass rest tests
+ // queryRestAPI(sliderClient, roles)
+
+ 5.times {
describe("Web UI is at $url")
sleep(60 *1000)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
index 3835330..f2f38e0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -20,8 +20,11 @@ package org.apache.slider.providers.agent
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.slider.agent.rest.RestAPIClientTestDelegates
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.types.ComponentInformation
import org.apache.slider.client.SliderClient
+import org.apache.slider.client.rest.SliderApplicationApiRestClient
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.PlacementPolicy
@@ -29,6 +32,8 @@ import org.junit.Test
import static org.apache.slider.common.params.Arguments.*
import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL
/**
* Tests an echo command
@@ -38,10 +43,12 @@ import static org.apache.slider.providers.agent.AgentKeys.*
class TestAgentAAEcho extends TestAgentEcho {
@Test
- public void testEchoOperation() throws Throwable {
+ public void testAgentEcho() throws Throwable {
assumeValidServerEnv()
-
- String clustername = createMiniCluster("",
+ def conf = configuration
+ conf.setBoolean(METRICS_LOGGING_ENABLED, true)
+ conf.setInt(METRICS_LOGGING_LOG_INTERVAL, 1)
+ String clustername = createMiniCluster("testaaecho",
configuration,
1,
1,
@@ -73,7 +80,6 @@ class TestAgentAAEcho extends TestAgentEcho {
true, true,
true)
postLaunchActions(launcher.service, clustername, echo, roles)
-
}
/**
@@ -105,6 +111,7 @@ class TestAgentAAEcho extends TestAgentEcho {
//expect the role count to be the same
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+ queryRestAPI(sliderClient, roles)
// flex size
// while running, ask for many more, expect them to still be outstanding
sleep(5000)
@@ -117,4 +124,18 @@ class TestAgentAAEcho extends TestAgentEcho {
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
}
+
+ protected void queryRestAPI(SliderClient sliderClient, Map<String, Integer> roles) {
+ initHttpTestSupport(sliderClient.config)
+ def applicationReport = sliderClient.applicationReport
+ def proxyAM = applicationReport.trackingUrl
+ GET(proxyAM)
+ describe "Proxy SliderRestClient Tests"
+ SliderApplicationApiRestClient restAPI =
+ new SliderApplicationApiRestClient(createUGIJerseyClient(), proxyAM)
+ def echoInfo = restAPI.getComponent(ECHO)
+ assert echoInfo.pendingAntiAffineRequestCount == 3
+ // no active requests ... there's no capacity
+ assert !echoInfo.isAARequestOutstanding
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index f14fc43..4ceeca0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -40,6 +40,7 @@ import static org.apache.slider.providers.agent.AgentKeys.*
@Slf4j
class TestAgentEcho extends AgentTestBase {
+ protected static final String ECHO = "echo"
File slider_core
String echo_py
File echo_py_path
@@ -59,7 +60,6 @@ class TestAgentEcho extends AgentTestBase {
agt_ver_path = new File(slider_core, agt_ver)
agt_conf = "agent.ini"
agt_conf_path = new File(slider_core, agt_conf)
-
}
@Override
@@ -68,7 +68,7 @@ class TestAgentEcho extends AgentTestBase {
}
@Test
- public void testEchoOperation() throws Throwable {
+ public void testAgentEcho() throws Throwable {
assumeValidServerEnv()
String clustername = createMiniCluster("",
@@ -81,7 +81,7 @@ class TestAgentEcho extends AgentTestBase {
validatePaths()
- def role = "echo"
+ def role = ECHO
Map<String, Integer> roles = [
(role): 2,
];
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
index 4721552..b4a7ac9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
@@ -33,8 +33,9 @@ public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
String USERNAME = "bigdataborat"
int WAIT_TIME = 120;
- String WAIT_TIME_ARG = WAIT_TIME.toString()
+ String WAIT_TIME_ARG = Integer.toString(WAIT_TIME)
String SLIDER_TEST_XML = "slider-test.xml"
+ String NODES_UPDATED_FLAG_METRIC = "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/python/agent.py
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/agent.py b/slider-core/src/test/python/agent.py
index 4177074..1c9ff5b 100644
--- a/slider-core/src/test/python/agent.py
+++ b/slider-core/src/test/python/agent.py
@@ -24,14 +24,21 @@ import time
from optparse import OptionParser
import os
+
# A representative Agent code for the embedded agent
def main():
- print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ print "Executing src/test/python/agent.py"
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
parser.add_option("--config", dest="conf_folder", help="conf folder")
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
+
(options, args) = parser.parse_args()
if options.log_folder:
@@ -43,9 +50,17 @@ def main():
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 30
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/agent/main.py b/slider-core/src/test/python/agent/main.py
index 116d179..1e851bb 100755
--- a/slider-core/src/test/python/agent/main.py
+++ b/slider-core/src/test/python/agent/main.py
@@ -26,7 +26,7 @@ import os
def main():
- print "Executing echo"
+ print "Executing src/test/python/agent/main.py"
try:
print 'Argument List: {0}'.format(str(sys.argv))
except AttributeError:
@@ -37,6 +37,8 @@ def main():
parser.add_option("--config", dest="conf_folder", help="conf folder")
parser.add_option('--command', dest='command', help='command to execute')
parser.add_option('--label', dest='label', help='label')
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
parser.add_option('--zk-quorum', dest='host:2181', help='zookeeper quorum')
parser.add_option('--zk-reg-path', dest='/register/org-apache-slider/cl1', help='zookeeper registry path')
@@ -51,9 +53,17 @@ def main():
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 30
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/python/echo.py
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/echo.py b/slider-core/src/test/python/echo.py
index ea5e8ce..1dabf1c 100644
--- a/slider-core/src/test/python/echo.py
+++ b/slider-core/src/test/python/echo.py
@@ -26,13 +26,18 @@ import os
def main():
- print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ print "Executing src/test/python/echo.py"
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
parser.add_option("--config", dest="conf_folder", help="conf folder")
parser.add_option('--command', dest='command', help='command to execute')
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
(options, args) = parser.parse_args()
if options.log_folder:
@@ -44,9 +49,17 @@ def main():
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 300
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/591ba99e/slider-core/src/test/python/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index 7f9cd23..b7251d3 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -24,7 +24,7 @@
</comment>
<version>0.1</version>
<type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
+ <minHadoopVersion>2.6.0</minHadoopVersion>
<components>
<component>
<name>hbase-rs</name>
[5/8] incubator-slider git commit: SLIDER-988 add mock test of
failure of AA container and re-request; fix any failures
Posted by st...@apache.org.
SLIDER-988 add mock test of failure of AA container and re-request; fix any failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/830864ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/830864ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/830864ff
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 830864ff701a7d1682c39def20dda909c3b4e7e5
Parents: 591ba99
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 17 19:55:59 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 17 19:55:59 2015 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 15 ++-
.../management/BoolMetricPredicate.java | 44 ++++++++
.../management/LongMetricFunction.java | 44 ++++++++
.../management/MetricsAndMonitoring.java | 51 ++++++++-
.../management/MetricsBindingService.java | 12 +-
.../appmaster/management/MetricsConstants.java | 2 +
.../management/PrefixedMetricsSet.java | 53 +++++++++
.../slider/server/appmaster/state/AppState.java | 69 +++++++-----
.../appmaster/state/OutstandingRequest.java | 6 +-
.../server/appmaster/state/RoleStatus.java | 82 ++++++++++----
.../TestMockAppStateAAOvercapacity.groovy | 110 +++++++++++++++++++
.../appstate/TestMockAppStateAAPlacement.groovy | 11 +-
.../TestRoleHistoryContainerEvents.groovy | 4 +-
.../model/mock/BaseMockAppStateTest.groovy | 14 ---
14 files changed, 438 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 5bf8622..eb7a9d5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -155,6 +155,7 @@ public final class SliderUtils {
* name of docker program
*/
public static final String DOCKER = "docker";
+ public static final int NODE_LIST_LIMIT = 10;
private SliderUtils() {
}
@@ -2468,7 +2469,7 @@ public final class SliderUtils {
* @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
*/
public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? +1 : ((x == y) ? 0 : -1);
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
}
public static String getSystemEnv(String property) {
@@ -2490,9 +2491,15 @@ public final class SliderUtils {
}
List<String> nodes = request.getNodes();
if (nodes != null) {
- buffer.append("Nodes = [")
- .append(join(nodes, ", ", false))
- .append("]; ");
+ buffer.append("Nodes = [ ");
+ int size = nodes.size();
+ for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) {
+ buffer.append(nodes.get(i)).append(' ');
+ }
+ if (size > NODE_LIST_LIMIT) {
+ buffer.append(String.format("...(total %d entries)", size));
+ }
+ buffer.append("]; ");
}
List<String> racks = request.getRacks();
if (racks != null) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
new file mode 100644
index 0000000..82bcd3a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+/**
+ * A metric which takes a predicate and returns 1 if the predicate evaluates
+ * to true. The predicate is evaluated whenever the metric is read.
+ */
+public class BoolMetricPredicate implements Metric, Gauge<Integer> {
+
+ private final Eval predicate;
+
+ public BoolMetricPredicate(Eval predicate) {
+ this.predicate = predicate;
+ }
+
+ @Override
+ public Integer getValue() {
+ return predicate.eval() ? 1: 0;
+ }
+
+ public interface Eval {
+ boolean eval();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
new file mode 100644
index 0000000..1de7345
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+/**
+ * A metric which takes a function to generate a long value.
+ * The function is evaluated whenever the metric is read.
+ */
+public class LongMetricFunction implements Metric, Gauge<Long> {
+
+ private final Eval function;
+
+ public LongMetricFunction(Eval function) {
+ this.function = function;
+ }
+
+ @Override
+ public Long getValue() {
+ return function.eval();
+ }
+
+ public interface Eval {
+ long eval();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
index cced42a..37a8935 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -20,9 +20,12 @@ package org.apache.slider.server.appmaster.management;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
import com.codahale.metrics.health.HealthCheckRegistry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -33,7 +36,8 @@ import java.util.concurrent.ConcurrentHashMap;
* Class for all metrics and monitoring
*/
public class MetricsAndMonitoring extends CompositeService {
-
+ protected static final Logger log =
+ LoggerFactory.getLogger(MetricsAndMonitoring.class);
public MetricsAndMonitoring(String name) {
super(name);
}
@@ -52,13 +56,15 @@ public class MetricsAndMonitoring extends CompositeService {
private final Map<String, MeterAndCounter> meterAndCounterMap
= new ConcurrentHashMap<>();
+ private final List<MetricSet> metricSets = new ArrayList<>();
+
/**
* List of recorded events
*/
private final List<RecordedEvent> eventHistory = new ArrayList<>(100);
public static final int EVENT_LIMIT = 1000;
-
+
public MetricRegistry getMetrics() {
return metrics;
}
@@ -74,6 +80,14 @@ public class MetricsAndMonitoring extends CompositeService {
super.serviceInit(conf);
}
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ for (MetricSet set : metricSets) {
+ unregister(set);
+ }
+ }
+
public MeterAndCounter getMeterAndCounter(String name) {
return meterAndCounterMap.get(name);
}
@@ -144,5 +158,38 @@ public class MetricsAndMonitoring extends CompositeService {
public synchronized List<RecordedEvent> cloneEventHistory() {
return new ArrayList<>(eventHistory);
}
+
+ /**
+ * Add a metric set for registering and deregistration on service stop
+ * @param metricSet metric set
+ */
+ public void addMetricSet(MetricSet metricSet) {
+ metricSets.add(metricSet);
+ metrics.registerAll(metricSet);
+ }
+
+ /**
+ * add a metric set, giving each entry a prefix
+ * @param prefix prefix (a trailing "." is automatically added)
+ * @param metricSet the metric set to register
+ */
+ public void addMetricSet(String prefix, MetricSet metricSet) {
+ addMetricSet(new PrefixedMetricsSet(prefix, metricSet));
+ }
+
+ /**
+ * Unregister a metric set; robust
+ * @param metricSet metric set to unregister
+ */
+ public void unregister(MetricSet metricSet) {
+ for (String s : metricSet.getMetrics().keySet()) {
+ try {
+ metrics.remove(s);
+ } catch (IllegalArgumentException e) {
+ // log but continue
+ log.info("Exception when trying to unregister {}", s, e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
index f8646bf..864a1cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
@@ -19,7 +19,9 @@
package org.apache.slider.server.appmaster.management;
import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.Slf4jReporter;
import com.google.common.base.Preconditions;
@@ -29,6 +31,9 @@ import org.apache.slider.server.services.workflow.ClosingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
@@ -79,7 +84,7 @@ public class MetricsBindingService extends CompositeService
JmxReporter jmxReporter;
jmxReporter = JmxReporter.forRegistry(metrics).build();
jmxReporter.start();
- addService(new ClosingService<JmxReporter>(jmxReporter));
+ addService(new ClosingService<>(jmxReporter));
// Ganglia
@@ -128,7 +133,7 @@ public class MetricsBindingService extends CompositeService
.convertDurationsTo(TimeUnit.MILLISECONDS)
.build();
reporter.start(interval, TimeUnit.MINUTES);
- addService(new ClosingService<ScheduledReporter>(reporter));
+ addService(new ClosingService<>(reporter));
summary.append(String.format(", SLF4J to log %s interval=%d",
logName, interval));
}
@@ -136,8 +141,11 @@ public class MetricsBindingService extends CompositeService
log.info(reportingDetails);
}
+
@Override
public String toString() {
return super.toString() + " " + reportingDetails;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
index 31a82a3..fa6bfc0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
@@ -53,4 +53,6 @@ public class MetricsConstants {
*/
public static final String CONTAINERS_START_FAILED = "containers.start-failed";
+ public static final String PREFIX_SLIDER_ROLES = "slider.roles.";
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
new file mode 100644
index 0000000..e9ad46a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * From an existing metrics set, generate a new metrics set with the
+ * prefix in front of every key.
+ *
+ * The prefix is added directly: if you want a '.' between prefix and metric
+ * keys, include it in the prefix.
+ */
+public class PrefixedMetricsSet implements MetricSet {
+
+ private final String prefix;
+ private final MetricSet source;
+
+ public PrefixedMetricsSet(String prefix, MetricSet source) {
+ this.prefix = prefix;
+ this.source = source;
+ }
+
+ @Override
+ public Map<String, Metric> getMetrics() {
+ Map<String, Metric> sourceMetrics = source.getMetrics();
+ Map<String, Metric> metrics = new HashMap<>(sourceMetrics.size());
+ for (Map.Entry<String, Metric> entry : sourceMetrics.entrySet()) {
+ metrics.put(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ return metrics;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2da5d36..49bc225 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -839,8 +839,11 @@ public class AppState {
}
RoleStatus roleStatus = new RoleStatus(providerRole);
roleStatusMap.put(priority, roleStatus);
- roles.put(providerRole.name, providerRole);
+ String name = providerRole.name;
+ roles.put(name, providerRole);
rolePriorityMap.put(priority, providerRole);
+ // register its entries
+ metricsAndMonitoring.addMetricSet(MetricsConstants.PREFIX_SLIDER_ROLES + name, roleStatus);
return roleStatus;
}
@@ -1511,7 +1514,6 @@ public class AppState {
public int exitStatus = 0;
public boolean unknownNode = false;
-
public String toString() {
final StringBuilder sb =
new StringBuilder("NodeCompletionResult{");
@@ -1969,7 +1971,8 @@ public class AppState {
expected = role.getDesired();
}
- log.info("Reviewing {} : expected {}", role, expected);
+ log.info("Reviewing {} : ", role);
+ log.debug("Expected {}, Delta: {}", expected, delta);
checkFailureThreshold(role);
if (expected < 0 ) {
@@ -1986,29 +1989,28 @@ public class AppState {
log.info("{}: Asking for {} more nodes(s) for a total of {} ", name, delta, expected);
if (role.isAntiAffinePlacement()) {
- // build one only if there is none outstanding, the role history knows
- // enough about the cluster to ask, and there is somewhere to place
- // the node
- if (role.getPendingAntiAffineRequests() == 0
- && !role.isAARequestOutstanding()
- && roleHistory.canPlaceAANodes()) {
- // log the number outstanding
- AMRMClient.ContainerRequest request = createAAContainerRequest(role);
- if (request != null) {
- log.info("Starting an anti-affine request sequence for {} nodes", delta);
- role.incPendingAntiAffineRequests(delta - 1);
- addContainerRequest(operations, request);
- } else {
- log.info("No location for anti-affine request");
+ long pending = delta;
+ if (roleHistory.canPlaceAANodes()) {
+ // build one only if there is none outstanding, the role history knows
+ // enough about the cluster to ask, and there is somewhere to place
+ // the node
+ if (!role.isAARequestOutstanding()) {
+ // no outstanding AA; try to place things
+ AMRMClient.ContainerRequest request = createAAContainerRequest(role);
+ if (request != null) {
+ pending--;
+ log.info("Starting an anti-affine request sequence for {} nodes; pending={}",
+ delta, pending);
+ addContainerRequest(operations, request);
+ } else {
+ log.info("No location for anti-affine request");
+ }
}
} else {
- if (roleHistory.canPlaceAANodes()) {
- log.info("Adding {} more anti-affine requests", delta);
- } else {
- log.warn("Awaiting node map before generating node requests");
- }
- role.incPendingAntiAffineRequests(delta);
+ log.warn("Awaiting node map before generating anti-affinity requests");
}
+ log.info("Setting pending to {}", pending);
+ role.setPendingAntiAffineRequests(pending);
} else {
for (int i = 0; i < delta; i++) {
@@ -2024,7 +2026,7 @@ public class AppState {
long excess = -delta;
// how many requests are outstanding? for AA roles, this includes pending
- long outstandingRequests = role.getRequested();
+ long outstandingRequests = role.getRequested() + role.getPendingAntiAffineRequests();
if (outstandingRequests > 0) {
// outstanding requests.
int toCancel = (int)Math.min(outstandingRequests, excess);
@@ -2084,13 +2086,11 @@ public class AppState {
roleId,
containersToRelease);
- //crop to the excess
-
- List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
+ // crop to the excess
+ List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
? containersToRelease.subList(0, (int)excess)
: containersToRelease;
-
// then build up a release operation, logging each container as released
for (RoleInstance possible : finalCandidates) {
log.info("Targeting for release: {}", possible);
@@ -2099,9 +2099,17 @@ public class AppState {
}
}
+ } else {
+ // actual + requested == desired
+ // there's a special case here: clear all pending AA requests
+ if (role.getPendingAntiAffineRequests() > 0) {
+ log.debug("Clearing outstanding pending AA requests");
+ role.setPendingAntiAffineRequests(0);
+ }
}
- // list of operations to execute
+ // there's now a list of operations to execute
+ log.debug("operations scheduled: {}; updated role: {}", operations.size(), role);
return operations;
}
@@ -2274,9 +2282,10 @@ public class AppState {
if (role.getPendingAntiAffineRequests() > 0) {
// still an outstanding AA request: need to issue a new one.
log.info("Asking for next container for AA role {}", roleName);
- role.decPendingAntiAffineRequests();
if (!addContainerRequest(operations, createAAContainerRequest(role))) {
log.info("No capacity in cluster for new requests");
+ } else {
+ role.decPendingAntiAffineRequests();
}
log.debug("Current AA role status {}", role);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 129fd4c..3a75f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -342,10 +342,12 @@ public final class OutstandingRequest extends RoleHostnamePair {
@Override
public String toString() {
- int requestRoleId = ContainerPriority.extractRole(getPriority());
boolean requestHasLocation = ContainerPriority.hasLocation(getPriority());
final StringBuilder sb = new StringBuilder("OutstandingRequest{");
- sb.append(super.toString());
+ sb.append("roleId=").append(roleId);
+ if (hostname != null) {
+ sb.append(", hostname='").append(hostname).append('\'');
+ }
sb.append(", node=").append(node);
sb.append(", hasLocation=").append(requestHasLocation);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 0fc3dc2..656f96c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -18,16 +18,21 @@
package org.apache.slider.server.appmaster.state;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.BoolMetric;
+import org.apache.slider.server.appmaster.management.BoolMetricPredicate;
import org.apache.slider.server.appmaster.management.LongGauge;
import java.io.Serializable;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Map;
/**
@@ -38,7 +43,7 @@ import java.util.Map;
* requires synchronization. Where synchronized access is good is that it allows for
* the whole instance to be locked, for updating multiple entries.
*/
-public final class RoleStatus implements Cloneable {
+public final class RoleStatus implements Cloneable, MetricSet {
private final String name;
@@ -48,32 +53,30 @@ public final class RoleStatus implements Cloneable {
private final int key;
private final ProviderRole providerRole;
- private final LongGauge desired = new LongGauge();
private final LongGauge actual = new LongGauge();
- private final LongGauge requested = new LongGauge();
- private final LongGauge releasing = new LongGauge();
- private final LongGauge failed = new LongGauge();
- private final LongGauge startFailed = new LongGauge();
- private final LongGauge started= new LongGauge();
private final LongGauge completed = new LongGauge();
- private final LongGauge totalRequested = new LongGauge();
- private final LongGauge preempted = new LongGauge(0);
- private final LongGauge nodeFailed = new LongGauge(0);
+ private final LongGauge desired = new LongGauge();
+ private final LongGauge failed = new LongGauge();
private final LongGauge failedRecently = new LongGauge(0);
private final LongGauge limitsExceeded = new LongGauge(0);
+ private final LongGauge nodeFailed = new LongGauge(0);
+ /** Number of AA requests queued. */
+ private final LongGauge pendingAntiAffineRequests = new LongGauge(0);
+ private final LongGauge preempted = new LongGauge(0);
+ private final LongGauge releasing = new LongGauge();
+ private final LongGauge requested = new LongGauge();
+ private final LongGauge started = new LongGauge();
+ private final LongGauge startFailed = new LongGauge();
+ private final LongGauge totalRequested = new LongGauge();
/** resource requirements */
private Resource resourceRequirements;
- /**
- * Number of AA requests queued. These should be reduced first on a
- * flex down.
- */
- private final LongGauge pendingAntiAffineRequests = new LongGauge(0);
/** any pending AA request */
private volatile OutstandingRequest outstandingAArequest = null;
+
private String failureMessage = "";
public RoleStatus(ProviderRole providerRole) {
@@ -81,7 +84,37 @@ public final class RoleStatus implements Cloneable {
this.name = providerRole.name;
this.key = providerRole.id;
}
-
+
+ @Override
+ public Map<String, Metric> getMetrics() {
+ Map<String, Metric> metrics = new HashMap<>(15);
+ metrics.put("actual", actual);
+ metrics.put("completed", completed );
+ metrics.put("desired", desired);
+ metrics.put("failed", failed);
+ metrics.put("limitsExceeded", limitsExceeded);
+ metrics.put("nodeFailed", nodeFailed);
+ metrics.put("preempted", preempted);
+ metrics.put("pendingAntiAffineRequests", pendingAntiAffineRequests);
+ metrics.put("releasing", releasing);
+ metrics.put("requested", requested);
+ metrics.put("preempted", preempted);
+ metrics.put("releasing", releasing );
+ metrics.put("requested", requested);
+ metrics.put("started", started);
+ metrics.put("startFailed", startFailed);
+ metrics.put("totalRequested", totalRequested);
+
+ metrics.put("outstandingAArequest",
+ new BoolMetricPredicate(new BoolMetricPredicate.Eval() {
+ @Override
+ public boolean eval() {
+ return isAARequestOutstanding();
+ }
+ }));
+ return metrics;
+ }
+
public String getName() {
return name;
}
@@ -157,11 +190,11 @@ public final class RoleStatus implements Cloneable {
}
/**
- * Get the request count. For AA roles, this includes pending ones.
+ * Get the request count.
* @return a count of requested containers
*/
public long getRequested() {
- return requested.get() + pendingAntiAffineRequests.get();
+ return requested.get();
}
public long incRequested() {
@@ -222,6 +255,14 @@ public final class RoleStatus implements Cloneable {
}
/**
+ * expose the predicate {@link #isAARequestOutstanding()} as an integer,
+ * which is very convenient in tests
+ * @return 1 if there is an outstanding request; 0 if not
+ */
+ public int getOutstandingAARequestCount() {
+ return isAARequestOutstanding()? 1: 0;
+ }
+ /**
* Note that a role failed, text will
* be used in any diagnostics if an exception
* is later raised.
@@ -350,7 +391,6 @@ public final class RoleStatus implements Cloneable {
*/
public long getDelta() {
long inuse = getActualAndRequested();
- //don't know how to view these. Are they in-use or not?
long delta = desired.get() - inuse;
if (delta < 0) {
//if we are releasing, remove the number that are already released.
@@ -366,7 +406,7 @@ public final class RoleStatus implements Cloneable {
* @return the size of the application when outstanding requests are included.
*/
public long getActualAndRequested() {
- return actual.get() + requested.get() + pendingAntiAffineRequests.get();
+ return actual.get() + requested.get();
}
@Override
@@ -499,7 +539,7 @@ public final class RoleStatus implements Cloneable {
public synchronized RoleStatistics getStatistics() {
RoleStatistics stats = new RoleStatistics();
- stats.activeAA = isAARequestOutstanding() ? 1: 0;
+ stats.activeAA = getOutstandingAARequestCount();
stats.actual = actual.get();
stats.desired = desired.get();
stats.failed = failed.get();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
new file mode 100644
index 0000000..7728748
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement with a cluster of size 1
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateAAOvercapacity extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 1
+
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 1)
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(aaRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testOvercapacityRecovery() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ "verify the state. kill the allocated container and review")
+ //more than expected
+ long desired = 3
+ aaRole.desired = desired
+ assert appState.roleHistory.canPlaceAANodes()
+
+ //first request
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert desired - 1 == aaRole.pendingAntiAffineRequests
+ List<AbstractRMOperation > operationsOut = []
+ // allocate and re-submit
+ def instances = submitOperations(operations, [], operationsOut)
+ assert 1 == instances.size()
+ assertAllContainersAA()
+
+ // expect an outstanding AA request to be unsatisfied
+ assert aaRole.actual < aaRole.desired
+ assert !aaRole.requested
+ assert !aaRole.AARequestOutstanding
+ assert desired - 1 == aaRole.pendingAntiAffineRequests
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+
+ // now lets trigger a failure
+ def nodemap = cloneNodemap()
+ assert nodemap.size() == 1
+
+ def instance = instances[0]
+ def cid = instance.containerId
+
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid,
+ LauncherExitCodes.EXIT_TASK_LAUNCH_FAILURE))
+ assert result.containerFailed
+
+ assert aaRole.failed == 1
+ assert aaRole.actual == 0
+ def availablePlacements = appState.getRoleHistory().findNodeForNewAAInstance(aaRole)
+ assert availablePlacements.size() == 1
+ describe "expecting a successful review with available placements of $availablePlacements"
+ operations = appState.reviewRequestAndReleaseNodes()
+ assert operations.size() == 1
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 749e4fc..3461e23 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -113,7 +113,9 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
aaRole.desired = 2
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
getSingleRequest(ops)
+ assert aaRole.requested == 1
assert aaRole.pendingAntiAffineRequests == 1
+ assert aaRole.actualAndRequested + aaRole.pendingAntiAffineRequests == aaRole.desired
// now trigger that flex up
aaRole.desired = 3
@@ -121,6 +123,13 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
// expect: no new reqests, pending count ++
List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
assert ops2.empty
+ assert aaRole.actual + aaRole.pendingAntiAffineRequests + aaRole.outstandingAARequestCount ==
+ aaRole.desired
+
+ // 1 outstanding
+ assert aaRole.actual == 0
+ assert aaRole.AARequestOutstanding
+ // and one AA
assert aaRole.pendingAntiAffineRequests == 2
assertAllContainersAA()
@@ -141,7 +150,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
@Test
- public void testAllocateFlexDown() throws Throwable {
+ public void testAllocateFlexDownDecrementsPending() throws Throwable {
// want multiple instances, so there will be iterations
aaRole.desired = 2
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index c8a82bd..ca42546 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -28,12 +28,10 @@ import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeId
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.*
import org.junit.Test
@@ -402,7 +400,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
int startSize = nodemap.size()
// now send a list of updated (failed) nodes event
- List<NodeReport> nodesUpdated = new ArrayList<NodeReport>();
+ List<NodeReport> nodesUpdated = new ArrayList<>();
NodeReport nodeReport = NodeReport.newInstance(
NodeId.newInstance(hostname, 0),
NodeState.LOST,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/830864ff/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index da1bcb9..a53e0be 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -401,20 +401,6 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
}
/**
- * Scan through all containers and assert that the assignment is AA
- * @param index role index
- */
- void assertAllContainersAAOld(String index) {
- def nodemap = stateAccess.nodeInformationSnapshot
- nodemap.each { name, info ->
- def nodeEntry = info.entries[index]
- assert nodeEntry == null ||
- (nodeEntry.live - nodeEntry.releasing + nodeEntry.starting) <= 1,
- "too many instances on node $name"
- }
- }
-
- /**
* Get the node information as a large JSON String
* @return
*/
[6/8] incubator-slider git commit: SLIDER-979 AM web UI to show state
of AA request
Posted by st...@apache.org.
SLIDER-979 AM web UI to show state of AA request
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b2b58d35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b2b58d35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b2b58d35
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: b2b58d35e624ba877f0db684c988e4e44ce289fb
Parents: 830864f
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 17 19:56:22 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 17 19:56:22 2015 +0000
----------------------------------------------------------------------
.../slider/api/types/ComponentInformation.java | 4 +-
.../server/appmaster/web/view/IndexBlock.java | 131 ++++++++++++++-----
.../server/appmaster/web/view/NavBlock.java | 1 -
.../slider/agent/rest/TestStandaloneREST.groovy | 3 +-
.../providers/agent/DemoAgentAAEcho.groovy | 8 +-
.../providers/agent/TestAgentAAEcho.groovy | 54 ++++++--
.../appmaster/web/view/TestIndexBlock.groovy | 6 +-
.../apache/slider/test/SliderTestUtils.groovy | 6 +-
slider-core/src/test/python/agent/main.py | 2 +-
.../src/test/resources/example-slider-test.xml | 70 ----------
10 files changed, 150 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index 3771988..c46a59f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -91,14 +91,14 @@ public class ComponentInformation {
sb.append(", failureMessage='").append(failureMessage).append('\'');
sb.append(", placementPolicy=").append(placementPolicy);
sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding);
- sb.append(", pendingAntiAffineRequestCount").append(pendingAntiAffineRequestCount);
+ sb.append(", pendingAntiAffineRequestCount=").append(pendingAntiAffineRequestCount);
sb.append(", priority=").append(priority);
sb.append(", releasing=").append(releasing);
sb.append(", requested=").append(requested);
sb.append(", started=").append(started);
sb.append(", startFailed=").append(startFailed);
sb.append(", totalRequested=").append(totalRequested);
- sb.append("container count='")
+ sb.append(", container count='")
.append(containers == null ? 0 : containers.size())
.append('\'');
sb.append('}');
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 8152f27..b3be3bf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -32,6 +32,7 @@ import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -40,7 +41,7 @@ import java.util.Map.Entry;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
/**
- *
+ * The main content on the Slider AM web page
*/
public class IndexBlock extends SliderHamletBlock {
private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
@@ -59,16 +60,13 @@ public class IndexBlock extends SliderHamletBlock {
@Override
protected void render(Block html) {
- final String providerName = getProviderName();
-
- doIndex(html, providerName);
+ doIndex(html, getProviderName());
}
// An extra method to make testing easier since you can't make an instance of Block
@VisibleForTesting
protected void doIndex(Hamlet html, String providerName) {
ClusterDescription clusterStatus = appState.getClusterStatus();
- RoleStatistics roleStats = appState.getRoleStatistics();
String name = clusterStatus.name;
if (name != null && (name.startsWith(" ") || name.endsWith(" "))) {
name = "'" + name + "'";
@@ -111,21 +109,27 @@ public class IndexBlock extends SliderHamletBlock {
.td("Application configuration path: ")
.td(clusterStatus.originConfigurationPath)
._();
- table1._()._();
+ table1._();
+ div._();
+ div = null;
+ DIV<Hamlet> containers = html.div("container_instances")
+ .h3("Component Instances");
- html.div("container_instances").h3("Component Instances");
+ int aaRoleWithNoSuitableLocations = 0;
+ int aaRoleWithOpenRequest = 0;
+ int roleWithOpenRequest = 0;
- Hamlet.TABLE<DIV<Hamlet>> table = div.table();
- Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> tr = table.thead().tr();
- trb(tr, "Component");
- trb(tr, "Desired");
- trb(tr, "Actual");
- trb(tr, "Outstanding Requests");
- trb(tr, "Failed");
- trb(tr, "Failed to start");
- trb(tr, "Placement");
- tr._()._();
+ Hamlet.TABLE<DIV<Hamlet>> table = containers.table();
+ Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> header = table.thead().tr();
+ trb(header, "Component");
+ trb(header, "Desired");
+ trb(header, "Actual");
+ trb(header, "Outstanding Requests");
+ trb(header, "Failed");
+ trb(header, "Failed to start");
+ trb(header, "Placement");
+ header._()._(); // tr & thead
List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
Collections.sort(roleStatuses, new RoleStatus.CompareByName());
@@ -134,13 +138,26 @@ public class IndexBlock extends SliderHamletBlock {
String nameUrl = apiPath(LIVE_COMPONENTS) + "/" + roleName;
String aatext;
if (status.isAntiAffinePlacement()) {
- int outstanding = status.isAARequestOutstanding() ? 1: 0;
+ boolean aaRequestOutstanding = status.isAARequestOutstanding();
int pending = (int)status.getPendingAntiAffineRequests();
- aatext = String.format("Anti-affine: %d outstanding %s, %d pending %s",
- outstanding, plural(outstanding, "request"),
- pending, plural(pending, "request"));
+ aatext = buildAADetails(aaRequestOutstanding, pending);
+ if (SliderUtils.isSet(status.getLabelExpression())) {
+ aatext += " (label: " + status.getLabelExpression() + ")";
+ }
+ if (pending > 0 && !aaRequestOutstanding) {
+ aaRoleWithNoSuitableLocations ++;
+ } else if (aaRequestOutstanding) {
+ aaRoleWithOpenRequest++;
+ }
} else {
- aatext = "";
+ if (SliderUtils.isSet(status.getLabelExpression())) {
+ aatext = "label: " + status.getLabelExpression();
+ } else {
+ aatext = "";
+ }
+ if (status.getRequested() > 0) {
+ roleWithOpenRequest ++;
+ }
}
table.tr()
.td().a(nameUrl, roleName)._()
@@ -153,23 +170,69 @@ public class IndexBlock extends SliderHamletBlock {
._();
}
- table._()._();
+ // empty row for some more spacing
+ table.tr()._();
+ // close table
+ table._();
+
+ containers._();
+ containers = null;
// some spacing
- html.p()._();
- html.p()._();
+ html.div()._();
+ html.div()._();
+
+ DIV<Hamlet> diagnostics = html.div("diagnostics");
+
+ List<String> statusEntries = new ArrayList<>(0);
+ if (roleWithOpenRequest > 0) {
+ statusEntries.add(String.format("%d %s with requests unsatisfiable by cluster",
+ roleWithOpenRequest, plural(roleWithOpenRequest, "component")));
+ }
+ if (aaRoleWithNoSuitableLocations > 0) {
+ statusEntries.add(String.format("%d anti-affinity %s no suitable nodes in the cluster",
+ aaRoleWithNoSuitableLocations,
+ plural(aaRoleWithNoSuitableLocations, "component has", "components have")));
+ }
+ if (aaRoleWithOpenRequest > 0) {
+ statusEntries.add(String.format("%d anti-affinity %s with requests unsatisfiable by cluster",
+ aaRoleWithOpenRequest,
+ plural(aaRoleWithOpenRequest, "component has", "components have")));
+
+ }
+ if (!statusEntries.isEmpty()) {
+ diagnostics.h3("Diagnostics");
+ Hamlet.TABLE<DIV<Hamlet>> diagnosticsTable = diagnostics.table();
+ for (String entry : statusEntries) {
+ diagnosticsTable.tr().td(entry)._();
+ }
+ diagnosticsTable._();
+ }
+ diagnostics._();
- html.div("provider_info").h3(providerName + " information");
- UL<DIV<Hamlet>> ul = div.ul();
+ DIV<Hamlet> provider_info = html.div("provider_info");
+ provider_info.h3(providerName + " information");
+ UL<Hamlet> ul = html.ul();
addProviderServiceOptions(providerService, ul, clusterStatus);
- ul._()._();
+ ul._();
+ provider_info._();
}
- private String plural(int n, String text) {
- return n == 1 ? text : (text + "s");
+ @VisibleForTesting
+ String buildAADetails(boolean outstanding, int pending) {
+ return String.format("Anti-affinity:%s %d pending %s",
+ (outstanding ? " 1 active request and" : ""),
+ pending, plural(pending, "request"));
+ }
+
+ private String plural(int n, String singular) {
+ return plural(n, singular, singular + "s");
+ }
+ private String plural(int n, String singular, String plural) {
+ return n == 1 ? singular : plural;
}
- private void trb(Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> tr,
+ private void trb(Hamlet.TR tr,
String text) {
tr.td().b(text)._();
}
@@ -184,9 +247,9 @@ public class IndexBlock extends SliderHamletBlock {
return null == createTime ? "N/A" : createTime;
}
- protected void addProviderServiceOptions(ProviderService providerService,
- UL<DIV<Hamlet>> ul, ClusterDescription clusterStatus) {
- Map<String, String> details = providerService.buildMonitorDetails(
+ protected void addProviderServiceOptions(ProviderService provider,
+ UL ul, ClusterDescription clusterStatus) {
+ Map<String, String> details = provider.buildMonitorDetails(
clusterStatus);
if (null == details) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
index 515b1a3..069d386 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
@@ -41,7 +41,6 @@ public class NavBlock extends SliderHamletBlock {
li().a(this.prefix(), "Overview")._().
li().a(relPath(CONTAINER_STATS), "Statistics")._().
li().a(relPath(CLUSTER_SPEC), "Specification")._().
- li().a(relPath(CLUSTER_SPEC), "Specification")._().
li().a(rootPath(SYSTEM_METRICS_JSON), "Metrics")._().
li().a(rootPath(SYSTEM_HEALTHCHECK), "Health")._().
li().a(rootPath(SYSTEM_THREADS), "Threads")._().
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 0bd1df0..b0ae102 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -87,8 +87,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
// using the metrics, await the first node status update.
// this should be from AM launch itself
- awaitGaugeValue(
- appendToURL(proxyAM, SYSTEM_METRICS_JSON),
+ awaitGaugeValue(proxyAM,
NODES_UPDATED_FLAG_METRIC,
1,
WEB_STARTUP_TIME * 2, 500)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
index 8606417..94e7320 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
@@ -31,15 +31,15 @@ class DemoAgentAAEcho extends TestAgentAAEcho {
SliderClient sliderClient,
String clustername,
String roleName,
- Map<String, Integer> roles) {
+ Map<String, Integer> roles,
+ String proxyAM) {
- def applicationReport = sliderClient.applicationReport
- def url = applicationReport.trackingUrl
+ def url = proxyAM
// spin repeating the URl in the logs so YARN chatter doesn't lose it
describe("Web UI is at $url")
// run the superclass rest tests
- // queryRestAPI(sliderClient, roles)
+ // queryRestAPI(sliderClient, roles, proxyAM)
5.times {
describe("Web UI is at $url")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
index f2f38e0..255dcaf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -20,20 +20,20 @@ package org.apache.slider.providers.agent
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.agent.rest.RestAPIClientTestDelegates
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.api.types.ComponentInformation
import org.apache.slider.client.SliderClient
import org.apache.slider.client.rest.SliderApplicationApiRestClient
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.server.appmaster.management.MetricsConstants
import org.junit.Test
import static org.apache.slider.common.params.Arguments.*
import static org.apache.slider.providers.agent.AgentKeys.*
import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED
import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_METRICS_JSON
/**
* Tests an echo command
@@ -75,11 +75,31 @@ class TestAgentAAEcho extends TestAgentEcho {
ARG_DEFINE,
SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false",
ARG_COMP_OPT, echo, TEST_RELAX_VERIFICATION, "true",
-
],
true, true,
true)
- postLaunchActions(launcher.service, clustername, echo, roles)
+ SliderClient sliderClient = launcher.service
+ initHttpTestSupport(sliderClient.config)
+ def applicationReport = sliderClient.applicationReport
+ def proxyAM = applicationReport.trackingUrl
+ try {
+ postLaunchActions(sliderClient, clustername, echo, roles, proxyAM)
+ } catch (Exception ex) {
+ logMetricsQuietly(proxyAM)
+ throw ex;
+ }
+ }
+
+ /**
+ * retrieve cluster metrics and log quietly
+ * @param proxyAM
+ */
+ protected void logMetricsQuietly(String proxyAM) {
+ try {
+ log.error prettyPrintAsJson(GET(proxyAM, SYSTEM_METRICS_JSON));
+ } catch (Exception ex) {
+ log.warn("failed to get AM", ex)
+ }
}
/**
@@ -94,16 +114,20 @@ class TestAgentAAEcho extends TestAgentEcho {
}
/**
- * Any actions to perform after starting the agent cluster
+ * Any actions to perform after starting the agent cluster.
+ * HTTP client operations will have been set up already.
* @param sliderClient client for the cluster
* @param clustername cluster name
* @param roleName name of the echo role
- * @parm original set of roles
+ * @param roles original set of roles
+ * @param proxyAM URl to proxy AM.
*/
- protected void postLaunchActions(SliderClient sliderClient,
+ protected void postLaunchActions(
+ SliderClient sliderClient,
String clustername,
String roleName,
- Map<String, Integer> roles) {
+ Map<String, Integer> roles,
+ String proxyAM) {
def onlyOneEcho = [(roleName): 1]
waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME)
//sleep a bit
@@ -111,7 +135,7 @@ class TestAgentAAEcho extends TestAgentEcho {
//expect the role count to be the same
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
- queryRestAPI(sliderClient, roles)
+ queryRestAPI(sliderClient, roles, proxyAM)
// flex size
// while running, ask for many more, expect them to still be outstanding
sleep(5000)
@@ -125,16 +149,18 @@ class TestAgentAAEcho extends TestAgentEcho {
}
- protected void queryRestAPI(SliderClient sliderClient, Map<String, Integer> roles) {
- initHttpTestSupport(sliderClient.config)
- def applicationReport = sliderClient.applicationReport
- def proxyAM = applicationReport.trackingUrl
+ protected void queryRestAPI(SliderClient sliderClient, Map<String, Integer> roles, String proxyAM) {
GET(proxyAM)
describe "Proxy SliderRestClient Tests"
SliderApplicationApiRestClient restAPI =
new SliderApplicationApiRestClient(createUGIJerseyClient(), proxyAM)
+ awaitGaugeValue(proxyAM,
+ MetricsConstants.PREFIX_SLIDER_ROLES + "echo.pendingAntiAffineRequests",
+ 2,
+ WEB_STARTUP_TIME * 2, 500)
+
def echoInfo = restAPI.getComponent(ECHO)
- assert echoInfo.pendingAntiAffineRequestCount == 3
+ assert echoInfo.pendingAntiAffineRequestCount == 2
// no active requests ... there's no capacity
assert !echoInfo.isAARequestOutstanding
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
index a4db705..de5fdc7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -128,12 +128,10 @@ public class TestIndexBlock extends BaseMockAppStateAATest {
Hamlet hamlet = new Hamlet(pw, 0, false);
- int level = hamlet.nestLevel();
indexBlock.doIndex(hamlet, "accumulo");
def body = sw.toString()
log.info(body)
- assertEquals(body, level, hamlet.nestLevel())
// verify role data came out
assert body.contains("role0")
assertContains(role0_desired, body)
@@ -146,8 +144,9 @@ public class TestIndexBlock extends BaseMockAppStateAATest {
assertContains(aarole_desired, body)
assertContains(aarole_actual, body)
- assertContains(aarole_requested, body)
+// assertContains(aarole_requested, body)
assertContains(aarole_failures, body)
+ assert body.contains(indexBlock.buildAADetails(true, aarole_pending))
// verify that the sorting took place
assert body.indexOf("role0") < body.indexOf("role1")
@@ -159,6 +158,5 @@ public class TestIndexBlock extends BaseMockAppStateAATest {
def assertContains(int ex, String html) {
assertStringContains(Integer.toString(ex), html)
-
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index ab81c46..5ef388a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -637,7 +637,6 @@ class SliderTestUtils extends Assert {
static UrlConnectionOperations connectionOperations
static UgiJerseyBinding jerseyBinding;
-
/**
* Static initializer of the connection operations
* @param conf config
@@ -1458,15 +1457,16 @@ class SliderTestUtils extends Assert {
/**
* Await a specific gauge being of the desired value
- * @param target target URL
+ * @param am URL of appmaster
* @param gauge gauge name
* @param desiredValue desired value
* @param timeout timeout in millis
* @param sleepDur sleep in millis
*/
- public void awaitGaugeValue(String target, String gauge, int desiredValue,
+ public void awaitGaugeValue(String am, String gauge, int desiredValue,
int timeout,
int sleepDur) {
+ String target = appendToURL(am, SYSTEM_METRICS_JSON)
def text = "Probe $target for gauge $gauge == $desiredValue"
repeatUntilSuccess(text,
this.&probeMetricGaugeValue,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/agent/main.py b/slider-core/src/test/python/agent/main.py
index 1e851bb..2eacc89 100755
--- a/slider-core/src/test/python/agent/main.py
+++ b/slider-core/src/test/python/agent/main.py
@@ -53,7 +53,7 @@ def main():
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- sleeptime = 30
+ sleeptime = 300
if options.sleep:
sleeptime = int(options.sleep)
if sleeptime > 0:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b2b58d35/slider-core/src/test/resources/example-slider-test.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/example-slider-test.xml b/slider-core/src/test/resources/example-slider-test.xml
index abf42f9..ee9fc59 100644
--- a/slider-core/src/test/resources/example-slider-test.xml
+++ b/slider-core/src/test/resources/example-slider-test.xml
@@ -33,12 +33,6 @@
</property>
<property>
- <name>slider.test.teardown.killall</name>
- <description>Kill all hbase/accumulo, etc processes on test teardown</description>
- <value>true</value>
- </property>
-
- <property>
<name>slider.test.thaw.wait.seconds</name>
<description>Time to wait for a start to work</description>
<value>60</value>
@@ -50,68 +44,4 @@
<value>60</value>
</property>
-
- <!-- Properties for the slider-hbase-provider only -not HBase-under-agent- -->
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.hbase.launch.wait.seconds</name>
- <description>Time to wait for the HBase application to be live</description>
- <value>180</value>
- </property>
-
- <property>
- <name>slider.test.hbase.home</name>
- <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1</value>
- <description>HBASE Home</description>
- </property>
-
- <property>
- <name>slider.test.hbase.tar</name>
- <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1-bin.tar.gz</value>
- <description>HBASE archive URI</description>
- </property>
-
- <!-- Properties for the slider-accumulo-provider only -not HBase-under-agent- -->
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.launch.wait.seconds</name>
- <description>Time to wait for the accumulo application to be live</description>
- <value>180</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.home</name>
- <value>/home/slider/accumulo</value>
- <description>Accumulo Home</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.tar</name>
- <value>/home/slider/Projects/accumulo/accumulo-1.6.0-bin.tar.gz</value>
- <description>Accumulo archive URI</description>
- </property>
-
- <property>
- <name>zk.home</name>
- <value>/home/slider/zookeeper</value>
- <description>Zookeeper home dir on target systems</description>
- </property>
-
- <property>
- <name>hadoop.home</name>
- <value>/home/slider/hadoop/</value>
- <description>Hadoop home dir on target systems</description>
- </property>
-
</configuration>
[2/8] incubator-slider git commit: SLIDER-985: regression,
TestBuildBasicAgent failing. (cause: increasing the limit on the #of
agents allowed)
Posted by st...@apache.org.
SLIDER-985: regression, TestBuildBasicAgent failing. (cause: increasing the limit on the #of agents allowed)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c776b1ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c776b1ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c776b1ad
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: c776b1ad45dbd5403e2f6b2824254ba0f0935c5d
Parents: 7278c39
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 16 14:01:35 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 16 14:01:50 2015 +0000
----------------------------------------------------------------------
.../slider/core/persist/ConfPersister.java | 6 ++-
.../slider/providers/agent/AgentTestBase.groovy | 16 ++----
.../providers/agent/TestAgentAAEcho.groovy | 34 +++++++++---
.../providers/agent/TestBuildBasicAgent.groovy | 57 +++++++++++---------
slider-core/src/test/python/metainfo.xml | 2 +-
5 files changed, 68 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
index 60717f6..9759205 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
@@ -101,7 +101,7 @@ public class ConfPersister {
* Make the persistent directory
* @throws IOException IO failure
*/
- private void mkPersistDir() throws IOException {
+ public void mkPersistDir() throws IOException {
coreFS.getFileSystem().mkdirs(persistDir);
}
@@ -165,13 +165,15 @@ public class ConfPersister {
* Acquire the writelock
* @throws IOException IO
* @throws LockAcquireFailedException
- * @throws FileNotFoundException if the target dir does not exist
+ * @throws FileNotFoundException if the target dir does not exist.
*/
@VisibleForTesting
boolean acquireReadLock() throws FileNotFoundException,
IOException,
LockAcquireFailedException {
if (!coreFS.getFileSystem().exists(persistDir)) {
+ // the dir is not there, so the data is not there, so there
+ // is nothing to read
throw new FileNotFoundException(persistDir.toString());
}
long now = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 5bf1c1f..0e5cd00 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -32,6 +32,7 @@ import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
import org.junit.Rule
import org.junit.rules.TemporaryFolder
+import org.slf4j.Logger
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
@@ -53,7 +54,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
*/
public static void assumeValidServerEnv() {
try {
- SliderUtils.validateSliderServerEnvironment(log, true)
+ SliderUtils.validateSliderServerEnvironment(log as Logger, true)
} catch (Exception e) {
skip(e.toString())
}
@@ -135,12 +136,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
boolean create,
boolean blockUntilRunning) {
-
- YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
create ? SliderActions.ACTION_CREATE : SliderActions.ACTION_BUILD,
@@ -166,11 +162,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
List<String> extraArgs,
boolean deleteExistingData) {
- YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
SliderActions.ACTION_UPDATE,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
index 0b89f47..80ff5a8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -52,9 +52,7 @@ class TestAgentAAEcho extends TestAgentEcho {
validatePaths()
def echo = "echo"
- Map<String, Integer> roles = [
- (echo): 2,
- ];
+ Map<String, Integer> roles = buildRoleMap(echo)
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
roles,
[
@@ -74,10 +72,33 @@ class TestAgentAAEcho extends TestAgentEcho {
],
true, true,
true)
- SliderClient sliderClient = launcher.service
+ postLaunchActions(launcher.service, clustername, echo, roles)
+
+ }
+ /**
+ * Build the role map to use when creating teh cluster
+ * @param roleName the name used for the echo role
+ * @return the map
+ */
+ protected Map<String, Integer> buildRoleMap(String roleName) {
+ [
+ (roleName): 2,
+ ];
+ }
- def onlyOneEcho = [(echo): 1]
+ /**
+ * Any actions to perform after starting the agent cluster
+ * @param sliderClient client for the cluster
+ * @param clustername cluster name
+ * @param roleName name of the echo role
+ * @parm original set of roles
+ */
+ protected void postLaunchActions(SliderClient sliderClient,
+ String clustername,
+ String roleName,
+ Map<String, Integer> roles) {
+ def onlyOneEcho = [(roleName): 1]
waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME)
//sleep a bit
sleep(5000)
@@ -91,9 +112,8 @@ class TestAgentAAEcho extends TestAgentEcho {
sliderClient.flex(clustername, onlyOneEcho);
// while running, flex it with no changes
- sliderClient.flex(clustername, [(echo): 3]);
+ sliderClient.flex(clustername, [(roleName): 3]);
sleep(1000)
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 264d260..60e9035 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -94,7 +94,9 @@ class TestBuildBasicAgent extends AgentTestBase {
1,
true,
false)
- buildAgentCluster("test_build_basic_agent_node_only",
+
+ def cluster01 = clustername + "_01"
+ buildAgentCluster(cluster01,
[(ROLE_NODE): 1],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -110,7 +112,9 @@ class TestBuildBasicAgent extends AgentTestBase {
def master = "hbase-master"
def rs = "hbase-rs"
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+
+ def cluster02 = clustername + "_02"
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(cluster02,
[
(ROLE_NODE): 1,
(master): 1,
@@ -149,10 +153,11 @@ class TestBuildBasicAgent extends AgentTestBase {
def rscomponent = resource.getMandatoryComponent(rs)
assert "5" == rscomponent.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
- // now create an instance with no role priority for the newnode role
+ describe "build a cluster with no role priority for the newnode role"
+
try {
- def name2 = clustername + "-2"
- buildAgentCluster(name2,
+ def cluster03 = clustername + "_03"
+ buildAgentCluster(cluster03,
[
(ROLE_NODE): 2,
"role3": 1,
@@ -166,14 +171,16 @@ class TestBuildBasicAgent extends AgentTestBase {
],
true, false,
false)
- failWithBuildSucceeding(name2, "no priority for one role")
+ failWithBuildSucceeding(cluster03, "no priority for one role")
} catch (BadConfigException expected) {
}
+ describe "build a cluster with the number of agents out of range"
try {
- launcher = buildAgentCluster(clustername + "-10",
+ def cluster04 = clustername + "_04"
+ launcher = buildAgentCluster(cluster04,
[
- (ROLE_NODE): 4,
+ (ROLE_NODE): 2000,
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -185,15 +192,18 @@ class TestBuildBasicAgent extends AgentTestBase {
],
true, false,
false)
- failWithBuildSucceeding(ROLE_NODE, "too many instances")
+ failWithBuildSucceeding(cluster04, "too many instances")
} catch (BadConfigException expected) {
- assert expected.message.contains("Expected minimum is 1 and maximum is 2")
- assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Expected minimum is 1 and maximum is")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "out of range.")
}
- //duplicate priorities
+
+ describe "build a cluster with duplicate priorities for roles"
try {
- def name3 = clustername + "-3"
- buildAgentCluster(name3,
+ def cluster05 = clustername + "_05"
+ buildAgentCluster(cluster05,
[
(ROLE_NODE): 5,
(master): 1,
@@ -206,16 +216,16 @@ class TestBuildBasicAgent extends AgentTestBase {
true, false,
false)
- failWithBuildSucceeding(name3, "duplicate priorities")
+ failWithBuildSucceeding(cluster05, "duplicate priorities")
} catch (BadConfigException expected) {
}
- def cluster4 = clustername + "-4"
+ def cluster06 = clustername + "_06"
def jvmopts = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
- buildAgentCluster(cluster4,
+ buildAgentCluster(cluster06,
[
(master): 1,
(rs): 5
@@ -235,7 +245,7 @@ class TestBuildBasicAgent extends AgentTestBase {
false)
//now we want to look at the value
- AggregateConf instanceDefinition = loadInstanceDefinition(cluster4)
+ AggregateConf instanceDefinition = loadInstanceDefinition(cluster06)
def opt = instanceDefinition.getAppConfOperations().getComponentOpt(
SliderKeys.COMPONENT_AM,
RoleKeys.JVM_OPTS,
@@ -245,8 +255,8 @@ class TestBuildBasicAgent extends AgentTestBase {
// now create an instance with no component options, hence no
// entry in the app config
- def name5 = clustername + "-5"
- buildAgentCluster(name5,
+ def name07 = clustername + "_07"
+ buildAgentCluster(name07,
[
"hbase-rs": 1,
],
@@ -403,9 +413,8 @@ class TestBuildBasicAgent extends AgentTestBase {
def rscomponent2 = resource2.getMandatoryComponent(rs)
assert "6" == rscomponent2.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
}
-
+
public AggregateConf loadInstanceDefinition(String name) {
- def cluster4
def sliderFS = createSliderFileSystem()
def dirPath = sliderFS.buildClusterDirPath(name)
ConfPersister persister = new ConfPersister(sliderFS, dirPath)
@@ -697,10 +706,8 @@ class TestBuildBasicAgent extends AgentTestBase {
}
public void failWithBuildSucceeding(String name, String reason) {
- def badArgs1
AggregateConf instanceDefinition = loadInstanceDefinition(name)
- log.error(
- "Build operation should have failed from $reason : \n$instanceDefinition")
+ log.error("Build operation should have failed from $reason : \n$instanceDefinition")
fail("Build operation should have failed from $reason")
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/python/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index 2a8c9e0..7f9cd23 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -51,7 +51,7 @@
<name>echo</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>200</maxInstanceCount>
+ <maxInstanceCount>100</maxInstanceCount>
<commandScript>
<script>echo.py</script>
<scriptType>PYTHON</scriptType>