You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by se...@apache.org on 2017/01/10 22:36:02 UTC
aurora git commit: Expose stats on ZooKeeper connection state
Repository: aurora
Updated Branches:
refs/heads/master eb3660d9e -> 3bdf0274c
Expose stats on ZooKeeper connection state
* zk_connection_state_STATE shows 1 if STATE is current connection state, otherwise 0.
* zk_connection_state_STATE_counter represents occurence times of the STATE since scheduler state
Bugs closed: AURORA-1838
Reviewed at https://reviews.apache.org/r/54624/
Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/3bdf0274
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/3bdf0274
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/3bdf0274
Branch: refs/heads/master
Commit: 3bdf0274c18b77df354d3e0bf138e25072ed4706
Parents: eb3660d
Author: Jing Chen <mi...@gmail.com>
Authored: Tue Jan 10 23:35:21 2017 +0100
Committer: Stephan Erb <se...@apache.org>
Committed: Tue Jan 10 23:35:21 2017 +0100
----------------------------------------------------------------------
.../CuratorServiceDiscoveryModule.java | 69 +++++++++++++++++++-
.../discovery/AbstractDiscoveryModuleTest.java | 4 ++
2 files changed, 72 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/aurora/blob/3bdf0274/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java b/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
index 999a542..ea167a8 100644
--- a/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/discovery/CuratorServiceDiscoveryModule.java
@@ -15,11 +15,13 @@ package org.apache.aurora.scheduler.discovery;
import java.net.InetSocketAddress;
import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Predicate;
import javax.inject.Singleton;
import com.google.common.base.Joiner;
+import com.google.common.base.Supplier;
import com.google.common.collect.FluentIterable;
import com.google.inject.Exposed;
import com.google.inject.PrivateModule;
@@ -32,6 +34,7 @@ import org.apache.aurora.common.io.Codec;
import org.apache.aurora.common.net.InetSocketAddressHelper;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.common.stats.StatsProvider;
import org.apache.aurora.common.thrift.ServiceInstance;
import org.apache.aurora.common.zookeeper.Credentials;
import org.apache.aurora.common.zookeeper.ServerSet;
@@ -41,7 +44,10 @@ import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.framework.listen.Listenable;
import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.curator.framework.state.ConnectionStateListener;
import org.apache.curator.retry.BoundedExponentialBackoffRetry;
import org.apache.curator.utils.PathUtils;
import org.apache.zookeeper.data.ACL;
@@ -58,6 +64,8 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
private final String discoveryPath;
private final ZooKeeperConfig zooKeeperConfig;
+ private ConnectionState currentState;
+
CuratorServiceDiscoveryModule(String discoveryPath, ZooKeeperConfig zooKeeperConfig) {
this.discoveryPath = PathUtils.validatePath(discoveryPath);
this.zooKeeperConfig = requireNonNull(zooKeeperConfig);
@@ -76,7 +84,8 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
CuratorFramework provideCuratorFramework(
ShutdownRegistry shutdownRegistry,
@ServiceDiscoveryBindings.ZooKeeper Iterable<InetSocketAddress> zooKeeperCluster,
- ACLProvider aclProvider) {
+ ACLProvider aclProvider,
+ StatsProvider statsProvider) {
String connectString =
FluentIterable.from(zooKeeperCluster)
@@ -87,6 +96,31 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
connectString = connectString + zooKeeperConfig.getChrootPath().get();
}
+ // export current connection state
+ for (ConnectionState connectionState : ConnectionState.values()) {
+ statsProvider.makeGauge(
+ zkConnectionGaugeName(connectionState),
+ new Supplier<Integer>() {
+ @Override
+ public Integer get() {
+ return connectionState.equals(currentState) ? 1 : 0;
+ }
+ }
+ );
+ }
+
+ // connection state counter
+ AtomicLong zkConnectionConnectedCounter =
+ statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.CONNECTED));
+ AtomicLong zkConnectionReadonlyCounter =
+ statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.READ_ONLY));
+ AtomicLong zkConnectionSuspendedCounter =
+ statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.SUSPENDED));
+ AtomicLong zkConnectionReconnectedCounter =
+ statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.RECONNECTED));
+ AtomicLong zkConnectionLostCounter =
+ statsProvider.makeCounter(zkConnectionStateCounterName(ConnectionState.LOST));
+
// This emulates the default BackoffHelper configuration used by the legacy commons/zookeeper
// stack. BackoffHelper is unbounded, this dies after around 5 minutes using the 10 retries.
// NB: BoundedExponentialBackoffRetry caps max retries at 29 if you send it a larger value.
@@ -110,6 +144,31 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
}
CuratorFramework curatorFramework = builder.build();
+ Listenable<ConnectionStateListener> connectionStateListener = curatorFramework
+ .getConnectionStateListenable();
+ connectionStateListener.addListener((CuratorFramework client, ConnectionState newState) -> {
+ currentState = newState;
+ switch (newState) {
+ case CONNECTED:
+ zkConnectionConnectedCounter.getAndIncrement();
+ break;
+ case READ_ONLY:
+ zkConnectionReadonlyCounter.getAndIncrement();
+ break;
+ case SUSPENDED:
+ zkConnectionSuspendedCounter.getAndIncrement();
+ break;
+ case RECONNECTED:
+ zkConnectionReconnectedCounter.getAndIncrement();
+ break;
+ case LOST:
+ zkConnectionLostCounter.getAndIncrement();
+ break;
+ default:
+ currentState = null;
+ break;
+ }
+ });
// TODO(John Sirois): It would be nice to use a Service to control the lifecycle here, but other
// services (org.apache.aurora.scheduler.http.JettyServerModule.RedirectMonitor) rely on this
@@ -182,4 +241,12 @@ class CuratorServiceDiscoveryModule extends PrivateModule {
SingletonService provideSingletonService(CuratorFramework client, Codec<ServiceInstance> codec) {
return new CuratorSingletonService(client, discoveryPath, MEMBER_TOKEN, codec);
}
+
+ private String zkConnectionStateCounterName(ConnectionState state) {
+ return zkConnectionGaugeName(state) + "_counter";
+ }
+
+ private String zkConnectionGaugeName(ConnectionState state) {
+ return "zk_connection_state_" + state;
+ }
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/3bdf0274/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java b/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
index d90192b..0f2121e 100644
--- a/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/discovery/AbstractDiscoveryModuleTest.java
@@ -25,11 +25,13 @@ import com.google.inject.Module;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.common.stats.StatsProvider;
import org.apache.aurora.common.testing.TearDownTestCase;
import org.apache.aurora.common.zookeeper.Credentials;
import org.apache.aurora.common.zookeeper.SingletonService;
import org.apache.aurora.common.zookeeper.ZooKeeperUtils;
import org.apache.aurora.scheduler.app.ServiceGroupMonitor;
+import org.apache.aurora.scheduler.testing.FakeStatsProvider;
import org.junit.Test;
import static org.junit.Assert.assertNotNull;
@@ -58,6 +60,8 @@ abstract class AbstractDiscoveryModuleTest extends TearDownTestCase {
bind(ServiceDiscoveryBindings.ZOO_KEEPER_ACL_KEY)
.toInstance(ZooKeeperUtils.OPEN_ACL_UNSAFE);
+ bind(StatsProvider.class).toInstance(new FakeStatsProvider());
+
bindExtraRequirements(binder());
}
},