You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bs...@apache.org on 2015/08/19 22:23:04 UTC
incubator-geode git commit: GEODE-77 kicked-out members were not
recognizing they were shunned
Repository: incubator-geode
Updated Branches:
refs/heads/feature/GEODE-77 c5c8565c6 -> c61fe3466
GEODE-77 kicked-out members were not recognizing they were shunned
Members kicked out of the system were not properly processing messages
from the coordinator telling them to shut down.
This also renames the new commands in the deadlock detector.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/c61fe346
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/c61fe346
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/c61fe346
Branch: refs/heads/feature/GEODE-77
Commit: c61fe3466f0507f76a487f03b662447a0e0b3011
Parents: c5c8565
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Wed Aug 19 13:22:34 2015 -0700
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Wed Aug 19 13:22:34 2015 -0700
----------------------------------------------------------------------
.../internal/deadlock/DeadlockDetector.java | 22 ++++++++++----------
.../internal/deadlock/DependencyGraph.java | 13 +++++++++---
.../deadlock/MessageDependencyMonitor.java | 4 +++-
.../membership/gms/membership/GMSJoinLeave.java | 15 +++++++++++--
.../gms/messages/LeaveRequestMessage.java | 9 +++++++-
.../cli/commands/MiscellaneousCommands.java | 2 +-
6 files changed, 46 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c61fe346/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DeadlockDetector.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DeadlockDetector.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DeadlockDetector.java
index 724b328..7999aea 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DeadlockDetector.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DeadlockDetector.java
@@ -318,12 +318,12 @@ public class DeadlockDetector {
System.out.println("system created by collectDependencies.");
System.out.println();
System.out.println("usage: ");
- System.out.println("[print | findDeepestGraph | findDeadlockOnly | findThread threadName ] file1 ...");
+ System.out.println("[print | findImpasse | findCycle | findObject objectName ] file1 ...");
System.out.println();
System.out.println("print - prints all dependencies and threads in the graph");
- System.out.println("findDeepestGraph - looks for either a deadlock or the longest call chain in the graph");
- System.out.println("findDeadlockOnly - looks for a deadlock in the distributed system");
- System.out.println("findThread - finds the given thread by name/partial name and builds a dependency graph around it");
+ System.out.println("findImpasse - looks for either a deadlock or the longest call chain in the graph");
+ System.out.println("findCycle - looks for a deadlock");
+ System.out.println("findObject - finds the given object (thread, lock, message) by name/partial name and finds all call chains leading to that object");
}
public static void main(String... args) throws Exception {
@@ -339,7 +339,7 @@ public class DeadlockDetector {
graph = loadGraphs(1, args);
System.out.println(prettyFormat(graph));
break;
- case "findDeadlockOnly":
+ case "findCycle":
graph = loadGraphs(1, args);
List<Dependency> cycle = graph.findCycle();
if (cycle == null) {
@@ -348,16 +348,16 @@ public class DeadlockDetector {
System.out.println("deadlocked threads: \n" + cycle);
}
break;
- case "findDeepestGraph":
+ case "findImpasse":
graph = loadGraphs(1, args);
- graph = graph.findDeepestGraph();
+ graph = graph.findLongestCallChain();
if (graph == null) {
- System.out.println("no deepest graph could be found!");
+ System.out.println("no long call chain could be found!");
} else {
- System.out.println("deepest graph: \n" + prettyFormat(graph));
+ System.out.println("longest call chain: \n" + prettyFormat(graph));
}
break;
- case "findThread":
+ case "findObject":
graph = loadGraphs(2, args);
List<DependencyGraph> graphs = graph.findDependenciesWith(args[1]);
if (graphs.isEmpty()) {
@@ -365,7 +365,7 @@ public class DeadlockDetector {
} else {
int numGraphs = graphs.size();
int i=0;
- System.out.println("findThread \"" + args[1]+"\n\n");
+ System.out.println("findObject \"" + args[1]+"\"\n\n");
for (DependencyGraph g: graphs) {
i += 1;
System.out.println("graph " + i + " of " + numGraphs + ":");
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c61fe346/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DependencyGraph.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DependencyGraph.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DependencyGraph.java
index 069eb7e..83d0169 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DependencyGraph.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/DependencyGraph.java
@@ -18,6 +18,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.gemstone.gemfire.distributed.internal.deadlock.MessageDependencyMonitor.MessageKey;
import com.gemstone.gemfire.internal.util.PluckStacks;
/**
@@ -100,7 +101,7 @@ public class DependencyGraph implements Serializable {
}
/**
- * This will find the deepest call chain in the graph. If a
+ * This will find the longest call chain in the graph. If a
* cycle is detected it will be returned. Otherwise all
* subgraphs are traversed to find the one that has the most
* depth. This usually indicates the thread that is blocking
@@ -109,7 +110,7 @@ public class DependencyGraph implements Serializable {
* The findDependenciesWith method can then be used to find all
* top-level threads that are blocked by the culprit.
*/
- public DependencyGraph findDeepestGraph() {
+ public DependencyGraph findLongestCallChain() {
int depth = 0;
DependencyGraph deepest = null;
@@ -177,7 +178,13 @@ public class DependencyGraph implements Serializable {
// dependers.
Set<Object> allDependants = new HashSet<>();
for (Dependency dep: edges) {
- allDependants.add(dep.dependsOn);
+ if ( (dep.dependsOn instanceof LocalThread) ) {
+ if (dep.depender instanceof MessageKey) {
+ allDependants.add(dep.dependsOn);
+ }
+ } else {
+ allDependants.add(dep.dependsOn);
+ }
}
List<DependencyGraph> result = new LinkedList<>();
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c61fe346/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/MessageDependencyMonitor.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/MessageDependencyMonitor.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/MessageDependencyMonitor.java
index f5f432e..2f66258 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/MessageDependencyMonitor.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/deadlock/MessageDependencyMonitor.java
@@ -89,8 +89,10 @@ public class MessageDependencyMonitor implements DependencyMonitor {
return heldResources;
}
- private static class MessageKey implements Serializable {
+ public static class MessageKey implements Serializable {
+ private static final long serialVersionUID = 414781046295505260L;
+
private final InternalDistributedMember myId;
private final int processorId;
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c61fe346/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index 606fdeb..d3075bb 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -248,6 +248,9 @@ logger.info("received join response {}", response);
* @param incomingRequest
*/
private void processJoinRequest(JoinRequestMessage incomingRequest) {
+
+ logger.info("received join request from {}", incomingRequest.getMemberID());
+
if (incomingRequest.getMemberID().getVersionObject().compareTo(Version.CURRENT) < 0) {
logger.warn("detected an attempt to start a peer using an older version of the product {}",
incomingRequest.getMemberID());
@@ -282,11 +285,19 @@ logger.info("received join response {}", response);
* @param incomingRequest
*/
private void processLeaveRequest(LeaveRequestMessage incomingRequest) {
+
+ logger.info("received leave request from {} for {}", incomingRequest.getSender(), incomingRequest.getMemberID());
+
NetView v = currentView;
if (logger.isDebugEnabled()) {
logger.debug("JoinLeave.processLeaveRequest invoked. isCoordinator="+isCoordinator+ "; isStopping="+isStopping
+"; cancelInProgress="+services.getCancelCriterion().isCancelInProgress());
}
+
+ if (incomingRequest.getMemberID().equals(this.localAddress)) {
+ logger.info("I am being told to leave the distributed system");
+ services.getManager().forceDisconnect(incomingRequest.getReason());
+ }
if (!isCoordinator && !isStopping && !services.getCancelCriterion().isCancelInProgress()) {
logger.debug("JoinLeave is checking to see if I should become coordinator");
NetView check = new NetView(v, v.getViewId()+1);
@@ -780,7 +791,7 @@ logger.info("received join response {}", response);
}
else {
logger.debug("JoinLeave sending a leave request to {}", view.getCoordinator());
- LeaveRequestMessage m = new LeaveRequestMessage(view.getCoordinator(), this.localAddress);
+ LeaveRequestMessage m = new LeaveRequestMessage(view.getCoordinator(), this.localAddress, "this member is shutting down");
services.getMessenger().send(m);
}
} // view.size
@@ -804,7 +815,7 @@ logger.info("received join response {}", response);
@Override
public void memberShutdown(DistributedMember mbr, String reason) {
if (this.isCoordinator) {
- LeaveRequestMessage msg = new LeaveRequestMessage(this.localAddress, (InternalDistributedMember)mbr);
+ LeaveRequestMessage msg = new LeaveRequestMessage(this.localAddress, (InternalDistributedMember)mbr, reason);
recordViewRequest(msg);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c61fe346/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/LeaveRequestMessage.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/LeaveRequestMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/LeaveRequestMessage.java
index 632e35f..61183a8 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/LeaveRequestMessage.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/LeaveRequestMessage.java
@@ -12,8 +12,9 @@ import com.gemstone.gemfire.internal.Version;
public class LeaveRequestMessage extends HighPriorityDistributionMessage {
private InternalDistributedMember memberID;
+ private String reason;
- public LeaveRequestMessage(InternalDistributedMember coord, InternalDistributedMember id) {
+ public LeaveRequestMessage(InternalDistributedMember coord, InternalDistributedMember id, String reason) {
super();
setRecipient(coord);
this.memberID = id;
@@ -36,6 +37,10 @@ public class LeaveRequestMessage extends HighPriorityDistributionMessage {
public InternalDistributedMember getMemberID() {
return memberID;
}
+
+ public String getReason() {
+ return reason;
+ }
@Override
public Version[] getSerializationVersions() {
@@ -45,11 +50,13 @@ public class LeaveRequestMessage extends HighPriorityDistributionMessage {
@Override
public void toData(DataOutput out) throws IOException {
DataSerializer.writeObject(memberID, out);
+ DataSerializer.writeString(reason, out);
}
@Override
public void fromData(DataInput in) throws IOException, ClassNotFoundException {
memberID = DataSerializer.readObject(in);
+ reason = DataSerializer.readString(in);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/c61fe346/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/cli/commands/MiscellaneousCommands.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/cli/commands/MiscellaneousCommands.java b/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/cli/commands/MiscellaneousCommands.java
index 20e64e7..852a507 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/cli/commands/MiscellaneousCommands.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/cli/commands/MiscellaneousCommands.java
@@ -578,7 +578,7 @@ public class MiscellaneousCommands implements CommandMarker {
Collection<Dependency> deadlock = dependencyGraph.findCycle();
DependencyGraph deepest = null;
if (deadlock == null) {
- deepest = dependencyGraph.findDeepestGraph();
+ deepest = dependencyGraph.findLongestCallChain();
if (deepest != null) {
deadlock = deepest.getEdges();
}