You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/02/02 17:24:53 UTC
[06/10] incubator-slider git commit: SLIDER-768: renaming attributes
to emphasise "outstanding"; mock tests to validate;
ClusterDescription exporting
SLIDER-768: renaming attributes to emphasise "outstanding"; mock tests to validate; ClusterDescription exporting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b75581b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b75581b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b75581b5
Branch: refs/heads/develop
Commit: b75581b526ad621a2a8432b4fcbcd86afb078e35
Parents: 7f2e0c2
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 2 14:37:34 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 2 14:37:34 2015 +0000
----------------------------------------------------------------------
.../apache/slider/api/ClusterDescription.java | 8 +++-
.../types/ApplicationLivenessInformation.java | 4 +-
.../appmaster/management/MetricsConstants.java | 27 +++++++++++
.../slider/server/appmaster/state/AppState.java | 48 +++++++++++++++-----
.../rest/RestAPIClientTestDelegates.groovy | 2 +-
.../appstate/TestMockAppStateFlexing.groovy | 25 +++++++++-
6 files changed, 97 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
index 8bcff0e..7db7e7f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.SliderProviderFactory;
@@ -204,7 +205,12 @@ public class ClusterDescription implements Cloneable {
* Status information
*/
public Map<String, Object> status;
-
+
+ /**
+ * Liveness information; the same as returned
+ * on the <code>live/liveness/</code> URL
+ */
+ public ApplicationLivenessInformation liveness;
/**
* Creator.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
index d4d4278..4dd2cb7 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
@@ -31,14 +31,14 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ApplicationLivenessInformation {
public boolean allRequestsSatisfied;
- public int requested;
+ public int requestsOutstanding;
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("ApplicationLivenessInformation{");
sb.append("allRequestsSatisfied=").append(allRequestsSatisfied);
- sb.append(", requested=").append(requested);
+ sb.append(", requestsOutstanding=").append(requestsOutstanding);
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
new file mode 100644
index 0000000..e55cf60
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+/**
+ * Constants used in slider for metrics registration and lookup
+ */
+public class MetricsConstants {
+ public static final String CONTAINERS_OUTSTANDING_REQUESTS =
+ "containers.outstanding-requests";
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 97a22b3..5a8b38b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import com.codahale.metrics.Counter;
+import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
@@ -67,6 +68,7 @@ import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.management.MetricsConstants;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.CancelRequestOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
@@ -224,8 +226,7 @@ public class AppState {
/**
* Track the number of requested Containers
*/
- private final Counter requestedContainerCount = new Counter();
-
+ private final Counter outstandingContainerRequests = new Counter();
/**
* Map of requested nodes. This records the command used to start it,
@@ -295,9 +296,17 @@ public class AppState {
* @param recordFactory factory for YARN records
* @param metricsAndMonitoring metrics and monitoring services
*/
- public AppState(AbstractRecordFactory recordFactory, MetricsAndMonitoring metricsAndMonitoring) {
+ public AppState(AbstractRecordFactory recordFactory,
+ MetricsAndMonitoring metricsAndMonitoring) {
this.recordFactory = recordFactory;
- this.metricsAndMonitoring = metricsAndMonitoring;
+ this.metricsAndMonitoring = metricsAndMonitoring;
+
+ // register any metrics
+ MetricRegistry metrics = metricsAndMonitoring.getMetrics();
+ metrics.register(
+ MetricRegistry.name(AppState.class,
+ MetricsConstants.CONTAINERS_OUTSTANDING_REQUESTS),
+ outstandingContainerRequests);
}
public int getFailedCountainerCount() {
@@ -381,8 +390,15 @@ public class AppState {
}
/**
- * Get the dynamcally created view of the cluster status
- * @return
+ * Get the current view of the cluster status.
+ * <p>
+ * Calls to {@link #refreshClusterStatus()} trigger a
+ * refresh of this field.
+ * <p>
+ * This is read-only
+ * to the extent that changes here do not trigger updates in the
+ * application state.
+ * @return the cluster status
*/
public synchronized ClusterDescription getClusterStatus() {
return clusterStatus;
@@ -617,7 +633,7 @@ public class AppState {
BadConfigException {
String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
int priority = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PRIORITY,
+ ResourceKeys.COMPONENT_PRIORITY,
priOpt, 0, 1, -1);
String placementOpt = component.getOption(
ResourceKeys.COMPONENT_PLACEMENT_POLICY,
@@ -1205,7 +1221,7 @@ public class AppState {
*/
protected void incrementRequestCount(RoleStatus role) {
role.incRequested();
- requestedContainerCount.inc();
+ outstandingContainerRequests.inc();
}
/**
@@ -1216,7 +1232,7 @@ public class AppState {
*/
protected void decrementRequestCount(RoleStatus role) {
role.decRequested();
- requestedContainerCount.dec();
+ outstandingContainerRequests.dec();
}
@@ -1603,13 +1619,16 @@ public class AppState {
return percentage;
}
+ /**
+ * Update the cluster description with the current application state
+ */
public void refreshClusterStatus() {
refreshClusterStatus(null);
}
/**
- * Update the cluster description with anything interesting
+ * Update the cluster description with the current application state
* @param providerStatus status from the provider for the cluster info section
*/
public synchronized ClusterDescription refreshClusterStatus(Map<String, String> providerStatus) {
@@ -1654,8 +1673,13 @@ public class AppState {
cd.statistics.put(rolename, stats);
}
+
Map<String, Integer> sliderstats = getLiveStatistics();
cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats);
+
+ // liveness
+ cd.liveness = getApplicationLivenessInformation();
+
return cd;
}
@@ -1665,8 +1689,8 @@ public class AppState {
*/
public ApplicationLivenessInformation getApplicationLivenessInformation() {
ApplicationLivenessInformation li = new ApplicationLivenessInformation();
- int outstanding = (int) requestedContainerCount.getCount();
- li.requested = outstanding;
+ int outstanding = (int) outstandingContainerRequests.getCount();
+ li.requestsOutstanding = outstanding;
li.allRequestsSatisfied = outstanding == 0;
return li;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy
index 3762d5d..029cb70 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/RestAPIClientTestDelegates.groovy
@@ -248,6 +248,6 @@ class RestAPIClientTestDelegates extends AbstractRestTestDelegate {
describe "Liveness:\n$liveness"
assert liveness.allRequestsSatisfied
- assert !liveness.requested
+ assert !liveness.requestsOutstanding
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b75581b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 1db500b..dcb862d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -39,9 +39,19 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
@Test
public void testFlexDuringLaunchPhase() throws Throwable {
+
+ // ask for one instance of role0
role0Status.desired = 1
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+
+ // at this point there's now one request in the list
+ assert 1 == ops.size()
+ // and in a liveness check, one outstanding
+ def liveness = appState.applicationLivenessInformation
+ assert 1 == liveness.requestsOutstanding
+ assert !liveness.allRequestsSatisfied
+
List<Container> allocations = engine.execute(ops)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> releases = []
@@ -54,6 +64,10 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
ops = appState.reviewRequestAndReleaseNodes()
assert ops.empty
+ liveness = appState.applicationLivenessInformation
+ assert 0 == liveness.requestsOutstanding
+ assert liveness.allRequestsSatisfied
+
//now this is the start point.
appState.containerStartSubmitted(target, ri);
@@ -69,8 +83,17 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
assert !ops.empty
+
+ // second scan will find the first run outstanding, so not re-issue
+ // any more container requests
List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
assert ops2.empty
+
+ // and in a liveness check, one outstanding
+ def liveness = appState.applicationLivenessInformation
+ assert 1 == liveness.requestsOutstanding
+ assert !liveness.allRequestsSatisfied
+
}
@@ -104,7 +127,7 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
// now shrink again
- role0Status.desired = r0 = 1
+ role0Status.desired = 1
completionResults = []
instances = createStartAndStopNodes(completionResults)
assert instances.size() == 0