You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/07 00:30:00 UTC
[18/22] incubator-slider git commit: SLIDER-965 RoleStatus and
AppState move to using LongGauges to store numbers in
SLIDER-965 RoleStatus and AppState move to using LongGauges to store numbers in
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ac98d826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ac98d826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ac98d826
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: ac98d826e0e0bc21590f8b25aa4366707b054de2
Parents: 5b7f6dd
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 6 21:17:25 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Nov 6 21:17:25 2015 +0000
----------------------------------------------------------------------
.../slider/api/proto/RestTypeMarshalling.java | 1 -
.../slider/api/types/ComponentInformation.java | 1 -
.../server/appmaster/management/LongGauge.java | 42 ++++++++++---
.../slider/server/appmaster/state/AppState.java | 42 +++++--------
.../server/appmaster/state/RoleStatus.java | 62 ++++++++++++++++----
5 files changed, 101 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 115405c..b7985e6 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -85,7 +85,6 @@ public class RestTypeMarshalling {
info.failureMessage = wire.getFailureMessage();
}
info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
- info.pendingAntiAffineRequest = info.pendingAntiAffineRequestCount > 0;
return info;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index 3b4b8bd..9d8a4ee 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -52,7 +52,6 @@ public class ComponentInformation {
public int requested;
public int failed, started, startFailed, completed, totalRequested;
public int nodeFailed, failedRecently, preempted;
- public boolean pendingAntiAffineRequest;
public int pendingAntiAffineRequestCount;
public String failureMessage;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
index 72a8805..ac9ac0e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
@@ -25,7 +25,9 @@ import java.util.concurrent.atomic.AtomicLong;
/**
* This is a {@link AtomicLong} which acts as a metrics gauge: its state can be exposed as
- * a management value.
+ * a metrics.
+ * It also exposes some of the same method names as the Codahale Counter class, so that
+ * it's easy to swap in.
*
*/
public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
@@ -45,22 +47,46 @@ public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
this(0);
}
-
+ /**
+ * Get the value as a metric
+ * @return current value
+ */
@Override
public Long getValue() {
return get();
}
+ public Long getCount() {
+ return get();
+ }
+
+ /**
+ * {@code ++}
+ */
+ public void inc() {
+ incrementAndGet();
+ }
+ /**
+ * {@code --}
+ */
+ public void dec() {
+ decrementAndGet();
+ }
+
/**
- * Decrement to the floor of 0.
- * There's checks to stop more than one thread being in this method at the time, but
- * that doesn't stop other operations on the value
+ * Decrement to the floor of 0. Operations in parallel may cause confusion here,
+ * but it will still never go below zero
* @param delta delta
* @return the current value
*/
- public synchronized long decToFloor(long delta) {
- long newval = Math.max(0L, get() - delta);
- set(newval);
+ public long decToFloor(long delta) {
+ long l = get();
+ long r = l - delta;
+ if (r < 0) {
+ r = 0;
+ }
+ // if this fails, the decrement has been lost
+ compareAndSet(l, r);
return get();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index c46177a..f6fe474 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import com.codahale.metrics.Counter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
@@ -62,6 +63,7 @@ import org.apache.slider.core.persist.AggregateConfSerDeser;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.LongGauge;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.MetricsConstants;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
@@ -194,33 +196,33 @@ public class AppState {
/**
* Counter for completed containers ( complete denotes successful or failed )
*/
- private final Counter completedContainerCount = new Counter();
+ private final LongGauge completedContainerCount = new LongGauge();
/**
* Count of failed containers
*/
- private final Counter failedContainerCount = new Counter();
+ private final LongGauge failedContainerCount = new LongGauge();
/**
* # of started containers
*/
- private final Counter startedContainers = new Counter();
+ private final LongGauge startedContainers = new LongGauge();
/**
* # of containers that failed to start
*/
- private final Counter startFailedContainerCount = new Counter();
+ private final LongGauge startFailedContainerCount = new LongGauge();
/**
* Track the number of surplus containers received and discarded
*/
- private final Counter surplusContainers = new Counter();
+ private final LongGauge surplusContainers = new LongGauge();
/**
* Track the number of requested Containers
*/
- private final Counter outstandingContainerRequests = new Counter();
+ private final LongGauge outstandingContainerRequests = new LongGauge();
/**
* Map of requested nodes. This records the command used to start it,
@@ -1211,22 +1213,10 @@ public class AppState {
}
/**
- * dec requested count of a role
- * <p>
- * Also updates application state counters.
- * @param role role to decrement
- */
- protected synchronized void decrementRequestCount(RoleStatus role) {
- role.decRequested();
- }
-
- /**
* Inc #of outstanding requests.
*/
private void incOutstandingContainerRequests() {
- synchronized (outstandingContainerRequests) {
- outstandingContainerRequests.inc();
- }
+ outstandingContainerRequests.inc();
}
/**
@@ -1700,7 +1690,7 @@ public class AppState {
*/
public ApplicationLivenessInformation getApplicationLivenessInformation() {
ApplicationLivenessInformation li = new ApplicationLivenessInformation();
- int outstanding = (int) outstandingContainerRequests.getCount();
+ int outstanding = outstandingContainerRequests.intValue();
li.requestsOutstanding = outstanding;
li.allRequestsSatisfied = outstanding <= 0;
return li;
@@ -1716,15 +1706,15 @@ public class AppState {
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE,
liveNodes.size());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
- (int)completedContainerCount.getCount());
+ completedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
- (int)failedContainerCount.getCount());
+ failedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED,
- (int)startedContainers.getCount());
+ startedContainers.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED,
- (int) startFailedContainerCount.getCount());
+ startFailedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS,
- (int)surplusContainers.getCount());
+ surplusContainers.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED,
completionOfUnknownContainerEvent.get());
return sliderstats;
@@ -2085,7 +2075,7 @@ public class AppState {
final RoleStatus role = lookupRoleStatus(container);
//dec requested count
- decrementRequestCount(role);
+ role.decRequested();
//inc allocated count -this may need to be dropped in a moment,
// but us needed to update the logic below
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ac98d826/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 4197c4f..cba963c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -18,6 +18,8 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
@@ -60,17 +62,17 @@ public final class RoleStatus implements Cloneable {
private final LongGauge failedRecently = new LongGauge(0);
private final LongGauge limitsExceeded = new LongGauge(0);
- /** flag set to true if there is an outstanding anti-affine request */
- private final AtomicBoolean pendingAARequest = new AtomicBoolean(false);
+ /** resource requirements */
+ private Resource resourceRequirements;
/**
* Number of AA requests queued. These should be reduced first on a
* flex down.
*/
- private int pendingAntiAffineRequestCount = 0;
+ private final LongGauge pendingAntiAffineRequests = new LongGauge(0);
/** any pending AA request */
- public OutstandingRequest outstandingAArequest = null;
+ private OutstandingRequest outstandingAArequest = null;
private String failureMessage = "";
@@ -163,9 +165,8 @@ public final class RoleStatus implements Cloneable {
return requested.incrementAndGet();
}
-
- public long cancel(long count) {
- return requested.decToFloor(count);
+ public void cancel(long count) {
+ requested.decToFloor(count);
}
public void decRequested() {
@@ -204,6 +205,10 @@ public final class RoleStatus implements Cloneable {
return limitsExceeded.get();
}
+ public long incPendingAntiAffineRequests(long v) {
+ return pendingAntiAffineRequests.addAndGet(v);
+ }
+
/**
* Note that a role failed, text will
* be used in any diagnostics if an exception
@@ -285,6 +290,22 @@ public final class RoleStatus implements Cloneable {
return nodeFailed.get();
}
+ public long getPendingAntiAffineRequests() {
+ return pendingAntiAffineRequests.get();
+ }
+
+ public void setPendingAntiAffineRequests(long pendingAntiAffineRequests) {
+ this.pendingAntiAffineRequests.set(pendingAntiAffineRequests);
+ }
+
+ public OutstandingRequest getOutstandingAArequest() {
+ return outstandingAArequest;
+ }
+
+ public void setOutstandingAArequest(OutstandingRequest outstandingAArequest) {
+ this.outstandingAArequest = outstandingAArequest;
+ }
+
/**
* Get the number of roles we are short of.
* nodes released are ignored.
@@ -321,7 +342,7 @@ public final class RoleStatus implements Cloneable {
", actual=" + actual +
", requested=" + requested +
", releasing=" + releasing +
- ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequestCount +
+ ", pendingAntiAffineRequestCount=" + pendingAntiAffineRequests +
", failed=" + failed +
", failed recently=" + failedRecently.get() +
", node failed=" + nodeFailed.get() +
@@ -376,8 +397,7 @@ public final class RoleStatus implements Cloneable {
info.failedRecently = failedRecently.intValue();
info.nodeFailed = nodeFailed.intValue();
info.preempted = preempted.intValue();
- info.pendingAntiAffineRequest = pendingAARequest.get();
- info.pendingAntiAffineRequestCount = pendingAntiAffineRequestCount;
+ info.pendingAntiAffineRequestCount = pendingAntiAffineRequests.intValue();
return info;
}
@@ -389,6 +409,14 @@ public final class RoleStatus implements Cloneable {
return providerRole.labelExpression;
}
+ public Resource getResourceRequirements() {
+ return resourceRequirements;
+ }
+
+ public void setResourceRequirements(Resource resourceRequirements) {
+ this.resourceRequirements = resourceRequirements;
+ }
+
/**
* Compare two role status entries by name
*/
@@ -410,5 +438,17 @@ public final class RoleStatus implements Cloneable {
return (o1.getKey() < o2.getKey() ? -1 : (o1.getKey() == o2.getKey() ? 0 : 1));
}
}
-
+
+ /**
+ * Given a resource, set its requirements to those this role needs
+ * @param resource resource to configure
+ * @return the resource
+ */
+ public Resource copyResourceRequirements(Resource resource) {
+ Preconditions.checkNotNull(resourceRequirements,
+ "Role resource requirements have not been set");
+ resource.setMemory(resourceRequirements.getMemory());
+ resource.setVirtualCores(resourceRequirements.getVirtualCores());
+ return resource;
+ }
}