You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2015/10/19 16:57:14 UTC
[1/2] cassandra git commit: Expose phi values from failure detector
via JMX
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 cf0c61b8e -> 7159a754a
Expose phi values from failure detector via JMX
patch by Ron Kuris and Ariel Weisberg; reviewed by Ariel Weisberg and
Brandon Williams for CASSANDRA-9526
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3cd75001
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3cd75001
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3cd75001
Branch: refs/heads/cassandra-3.0
Commit: 3cd750012e0cf3b55442c036627fb032c29c16bc
Parents: 0bfa26d
Author: Ron Kuris <ro...@gmail.com>
Authored: Mon Oct 19 15:50:58 2015 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Oct 19 15:50:58 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/gms/FailureDetector.java | 82 +++++++++++++++-----
.../cassandra/gms/FailureDetectorMBean.java | 5 ++
.../org/apache/cassandra/tools/NodeProbe.java | 15 +++-
.../org/apache/cassandra/tools/NodeTool.java | 3 +-
.../tools/nodetool/FailureDetectorInfo.java | 46 +++++++++++
6 files changed, 132 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd75001/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0904559..3eff22c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
2.2.4
+ * Expose phi values from failure detector via JMX and tweak debug
+ and trace logging (CASSANDRA-9526)
* Fix RangeNamesQueryPager (CASSANDRA-10509)
* Deprecate Pig support (CASSANDRA-10542)
* Reduce contention getting instances of CompositeType (CASSANDRA-10433)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd75001/src/java/org/apache/cassandra/gms/FailureDetector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/FailureDetector.java b/src/java/org/apache/cassandra/gms/FailureDetector.java
index 861a853..c563872 100644
--- a/src/java/org/apache/cassandra/gms/FailureDetector.java
+++ b/src/java/org/apache/cassandra/gms/FailureDetector.java
@@ -22,10 +22,13 @@ import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import javax.management.MBeanServer;
import javax.management.ObjectName;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,6 +49,7 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
public static final String MBEAN_NAME = "org.apache.cassandra.net:type=FailureDetector";
private static final int SAMPLE_SIZE = 1000;
protected static final long INITIAL_VALUE_NANOS = TimeUnit.NANOSECONDS.convert(getInitialValue(), TimeUnit.MILLISECONDS);
+ private static final int DEBUG_PERCENTAGE = 80; // if the phi is larger than this percentage of the max, log a debug message
private static final long DEFAULT_MAX_PAUSE = 5000L * 1000000L; // 5 seconds
private static final long MAX_LOCAL_PAUSE_IN_NANOS = getMaxLocalPause();
private long lastInterpret = System.nanoTime();
@@ -71,8 +75,8 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
// change.
private final double PHI_FACTOR = 1.0 / Math.log(10.0); // 0.434...
- private final Map<InetAddress, ArrivalWindow> arrivalSamples = new Hashtable<InetAddress, ArrivalWindow>();
- private final List<IFailureDetectionEventListener> fdEvntListeners = new CopyOnWriteArrayList<IFailureDetectionEventListener>();
+ private final ConcurrentHashMap<InetAddress, ArrivalWindow> arrivalSamples = new ConcurrentHashMap<>();
+ private final List<IFailureDetectionEventListener> fdEvntListeners = new CopyOnWriteArrayList<>();
public FailureDetector()
{
@@ -148,6 +152,34 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
return count;
}
+ @Override
+ public TabularData getPhiValues() throws OpenDataException
+ {
+ final CompositeType ct = new CompositeType("Node", "Node",
+ new String[]{"Endpoint", "PHI"},
+ new String[]{"IP of the endpoint", "PHI value"},
+ new OpenType[]{SimpleType.STRING, SimpleType.DOUBLE});
+ final TabularDataSupport results = new TabularDataSupport(new TabularType("PhiList", "PhiList", ct, new String[]{"Endpoint"}));
+
+ for (final Map.Entry<InetAddress, ArrivalWindow> entry : arrivalSamples.entrySet())
+ {
+ final ArrivalWindow window = entry.getValue();
+ if (window.mean() > 0)
+ {
+ final double phi = window.getLastReportedPhi();
+ if (phi != Double.MIN_VALUE)
+ {
+ // returned values are scaled by PHI_FACTOR so that the are on the same scale as PhiConvictThreshold
+ final CompositeData data = new CompositeDataSupport(ct,
+ new String[]{"Endpoint", "PHI"},
+ new Object[]{entry.getKey().toString(), phi * PHI_FACTOR});
+ results.put(data);
+ }
+ }
+ }
+ return results;
+ }
+
public String getEndpointState(String address) throws UnknownHostException
{
StringBuilder sb = new StringBuilder();
@@ -219,8 +251,6 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
public void report(InetAddress ep)
{
- if (logger.isTraceEnabled())
- logger.trace("reporting {}", ep);
long now = System.nanoTime();
ArrivalWindow heartbeatWindow = arrivalSamples.get(ep);
if (heartbeatWindow == null)
@@ -228,12 +258,17 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
// avoid adding an empty ArrivalWindow to the Map
heartbeatWindow = new ArrivalWindow(SAMPLE_SIZE);
heartbeatWindow.add(now, ep);
- arrivalSamples.put(ep, heartbeatWindow);
+ heartbeatWindow = arrivalSamples.putIfAbsent(ep, heartbeatWindow);
+ if (heartbeatWindow != null)
+ heartbeatWindow.add(now, ep);
}
else
{
heartbeatWindow.add(now, ep);
}
+
+ if (logger.isTraceEnabled() && heartbeatWindow != null)
+ logger.info("Average for {} is {}", ep, heartbeatWindow.mean());
}
public void interpret(InetAddress ep)
@@ -263,13 +298,22 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
if (PHI_FACTOR * phi > getPhiConvictThreshold())
{
- logger.trace("notifying listeners that {} is down", ep);
- logger.trace("intervals: {} mean: {}", hbWnd, hbWnd.mean());
+ if (logger.isTraceEnabled())
+ logger.trace("Node {} phi {} > {}; intervals: {} mean: {}", new Object[]{ep, PHI_FACTOR * phi, getPhiConvictThreshold(), hbWnd, hbWnd.mean()});
for (IFailureDetectionEventListener listener : fdEvntListeners)
{
listener.convict(ep, phi);
}
}
+ else if (logger.isDebugEnabled() && (PHI_FACTOR * phi * DEBUG_PERCENTAGE / 100.0 > getPhiConvictThreshold()))
+ {
+ logger.debug("PHI for {} : {}", ep, phi);
+ }
+ else if (logger.isTraceEnabled())
+ {
+ logger.trace("PHI for {} : {}", ep, phi);
+ logger.trace("mean for {} : {}", ep, hbWnd.mean());
+ }
}
public void forceConviction(InetAddress ep)
@@ -312,10 +356,6 @@ public class FailureDetector implements IFailureDetector, FailureDetectorMBean
sb.append("-----------------------------------------------------------------------");
return sb.toString();
}
-
- public static void main(String[] args)
- {
- }
}
/*
@@ -372,12 +412,7 @@ class ArrivalWindow
private static final Logger logger = LoggerFactory.getLogger(ArrivalWindow.class);
private long tLast = 0L;
private final ArrayBackedBoundedStats arrivalIntervals;
-
- // this is useless except to provide backwards compatibility in phi_convict_threshold,
- // because everyone seems pretty accustomed to the default of 8, and users who have
- // already tuned their phi_convict_threshold for their own environments won't need to
- // change.
- private final double PHI_FACTOR = 1.0 / Math.log(10.0);
+ private double lastReportedPhi = Double.MIN_VALUE;
// in the event of a long partition, never record an interval longer than the rpc timeout,
// since if a host is regularly experiencing connectivity problems lasting this long we'd
@@ -411,9 +446,14 @@ class ArrivalWindow
{
long interArrivalTime = (value - tLast);
if (interArrivalTime <= MAX_INTERVAL_IN_NANO)
+ {
arrivalIntervals.add(interArrivalTime);
+ logger.trace("Reporting interval time of {} for {}", interArrivalTime, ep);
+ }
else
+ {
logger.debug("Ignoring interval time of {} for {}", interArrivalTime, ep);
+ }
}
else
{
@@ -435,7 +475,13 @@ class ArrivalWindow
{
assert arrivalIntervals.mean() > 0 && tLast > 0; // should not be called before any samples arrive
long t = tnow - tLast;
- return t / mean();
+ lastReportedPhi = t / mean();
+ return lastReportedPhi;
+ }
+
+ double getLastReportedPhi()
+ {
+ return lastReportedPhi;
}
public String toString()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd75001/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java b/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
index 45250b4..23fae3a 100644
--- a/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
+++ b/src/java/org/apache/cassandra/gms/FailureDetectorMBean.java
@@ -20,6 +20,9 @@ package org.apache.cassandra.gms;
import java.net.UnknownHostException;
import java.util.Map;
+import javax.management.openmbean.OpenDataException;
+import javax.management.openmbean.TabularData;
+
public interface FailureDetectorMBean
{
public void dumpInterArrivalTimes();
@@ -37,4 +40,6 @@ public interface FailureDetectorMBean
public int getDownEndpointCount();
public int getUpEndpointCount();
+
+ public TabularData getPhiValues() throws OpenDataException;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd75001/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index da403ab..62795b5 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -27,7 +27,6 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.rmi.server.RMIClientSocketFactory;
import java.rmi.server.RMISocketFactory;
-import java.text.SimpleDateFormat;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
@@ -522,7 +521,7 @@ public class NodeProbe implements AutoCloseable
/**
* Take a snapshot of all column family from different keyspaces.
- *
+ *
* @param snapshotName
* the name of the snapshot.
* @param columnFamilyList
@@ -1277,6 +1276,18 @@ public class NodeProbe implements AutoCloseable
}
}
}
+
+ public TabularData getFailureDetectorPhilValues()
+ {
+ try
+ {
+ return fdProxy.getPhiValues();
+ }
+ catch (OpenDataException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
}
class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd75001/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index da3560d..175b325 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -129,7 +129,8 @@ public class NodeTool
TpStats.class,
TopPartitions.class,
SetLoggingLevel.class,
- GetLoggingLevels.class
+ GetLoggingLevels.class,
+ FailureDetectorInfo.class
);
Cli.CliBuilder<Runnable> builder = Cli.builder("nodetool");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3cd75001/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java b/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java
new file mode 100644
index 0000000..72c109a
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/FailureDetectorInfo.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import io.airlift.command.Command;
+
+import java.util.List;
+
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.TabularData;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "failuredetector", description = "Shows the failure detector information for the cluster")
+public class FailureDetectorInfo extends NodeToolCmd
+{
+ @Override
+ public void execute(NodeProbe probe)
+ {
+ TabularData data = probe.getFailureDetectorPhilValues();
+ System.out.printf("%10s,%16s\n", "Endpoint", "Phi");
+ for (Object o : data.keySet())
+ {
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ CompositeData datum = data.get(((List) o).toArray(new Object[((List) o).size()]));
+ System.out.printf("%10s,%16.8f\n",datum.get("Endpoint"), datum.get("PHI"));
+ }
+ }
+}
+
[2/2] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by al...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7159a754
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7159a754
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7159a754
Branch: refs/heads/cassandra-3.0
Commit: 7159a754a5fcaa1b78a53089267ea7506e32e81d
Parents: cf0c61b 3cd7500
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Mon Oct 19 15:57:04 2015 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Oct 19 15:57:04 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../apache/cassandra/gms/FailureDetector.java | 82 +++++++++++++++-----
.../cassandra/gms/FailureDetectorMBean.java | 5 ++
.../org/apache/cassandra/tools/NodeProbe.java | 14 +++-
.../org/apache/cassandra/tools/NodeTool.java | 3 +-
.../tools/nodetool/FailureDetectorInfo.java | 46 +++++++++++
6 files changed, 133 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7159a754/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 9b86ebd,3eff22c..5a118ca
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,31 -1,18 +1,34 @@@
-2.2.4
+3.0
++Merged from 2.2:
+ * Expose phi values from failure detector via JMX and tweak debug
+ and trace logging (CASSANDRA-9526)
- * Fix RangeNamesQueryPager (CASSANDRA-10509)
- * Deprecate Pig support (CASSANDRA-10542)
- * Reduce contention getting instances of CompositeType (CASSANDRA-10433)
Merged from 2.1:
* AssertionError: attempted to delete non-existing file CommitLog (CASSANDRA-10377)
- * (cqlsh) Distinguish negative and positive infinity in output (CASSANDRA-10523)
- * (cqlsh) allow custom time_format for COPY TO (CASSANDRA-8970)
- * Don't allow startup if the node's rack has changed (CASSANDRA-10242)
- * (cqlsh) show partial trace if incomplete after max_trace_wait (CASSANDRA-7645)
-2.2.3
+3.0-rc2
+ * Fix SELECT DISTINCT queries between 2.2.2 nodes and 3.0 nodes (CASSANDRA-10473)
+ * Remove circular references in SegmentedFile (CASSANDRA-10543)
+ * Ensure validation of indexed values only occurs once per-partition (CASSANDRA-10536)
+ * Fix handling of static columns for range tombstones in thrift (CASSANDRA-10174)
+ * Support empty ColumnFilter for backward compatility on empty IN (CASSANDRA-10471)
+ * Remove Pig support (CASSANDRA-10542)
+ * Fix LogFile throws Exception when assertion is disabled (CASSANDRA-10522)
+ * Revert CASSANDRA-7486, make CMS default GC, move GC config to
+ conf/jvm.options (CASSANDRA-10403)
+ * Fix TeeingAppender causing some logs to be truncated/empty (CASSANDRA-10447)
+ * Allow EACH_QUORUM for reads (CASSANDRA-9602)
+ * Fix potential ClassCastException while upgrading (CASSANDRA-10468)
+ * Fix NPE in MVs on update (CASSANDRA-10503)
+ * Only include modified cell data in indexing deltas (CASSANDRA-10438)
+ * Do not load keyspace when creating sstable writer (CASSANDRA-10443)
+ * If node is not yet gossiping write all MV updates to batchlog only (CASSANDRA-10413)
+ * Re-populate token metadata after commit log recovery (CASSANDRA-10293)
+ * Provide additional metrics for materialized views (CASSANDRA-10323)
+ * Flush system schema tables after local schema changes (CASSANDRA-10429)
+Merged from 2.2:
+ * Reduce contention getting instances of CompositeType (CASSANDRA-10433)
+ * Fix the regression when using LIMIT with aggregates (CASSANDRA-10487)
* Avoid NoClassDefFoundError during DataDescriptor initialization on windows (CASSANDRA-10412)
* Preserve case of quoted Role & User names (CASSANDRA-10394)
* cqlsh pg-style-strings broken (CASSANDRA-10484)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7159a754/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/NodeProbe.java
index d07a54c,62795b5..b4821f5
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@@ -527,18 -521,18 +527,18 @@@ public class NodeProbe implements AutoC
/**
* Take a snapshot of all column family from different keyspaces.
- *
+ *
* @param snapshotName
* the name of the snapshot.
- * @param columnFamilyList
+ * @param tableList
* list of columnfamily from different keyspace in the form of ks1.cf1 ks2.cf2
*/
- public void takeMultipleColumnFamilySnapshot(String snapshotName, String... columnFamilyList)
+ public void takeMultipleTableSnapshot(String snapshotName, String... tableList)
throws IOException
{
- if (null != columnFamilyList && columnFamilyList.length != 0)
+ if (null != tableList && tableList.length != 0)
{
- ssProxy.takeMultipleColumnFamilySnapshot(snapshotName, columnFamilyList);
+ ssProxy.takeMultipleTableSnapshot(snapshotName, tableList);
}
else
{
@@@ -1293,17 -1277,17 +1293,29 @@@
}
}
+ public void replayBatchlog() throws IOException
+ {
+ try
+ {
+ bmProxy.forceBatchlogReplay();
+ }
+ catch (Exception e)
+ {
+ throw new IOException(e);
+ }
+ }
++
+ public TabularData getFailureDetectorPhilValues()
+ {
+ try
+ {
+ return fdProxy.getPhiValues();
+ }
+ catch (OpenDataException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
}
class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7159a754/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/NodeTool.java
index afee17a,175b325..e4bed4c
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@@ -131,8 -130,7 +131,9 @@@ public class NodeToo
TopPartitions.class,
SetLoggingLevel.class,
GetLoggingLevels.class,
+ DisableHintsForDC.class,
- EnableHintsForDC.class
++ EnableHintsForDC.class,
+ FailureDetectorInfo.class
);
Cli.CliBuilder<Runnable> builder = Cli.builder("nodetool");