svn commit: r1584271 - in /lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud: ElectionContext.java Overseer.java OverseerCollectionProcessor.java

[no...@apache.org]

Author: noble
Date: Thu Apr  3 07:07:50 2014
New Revision: 1584271

URL: http://svn.apache.org/r1584271
Log:
SOLR-5859 improved logging, and fix a potential bug

Modified:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1584271&r1=1584270&r2=1584271&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Thu Apr  3 07:07:50 2014
@@ -68,6 +68,7 @@ public abstract class ElectionContext {
   
   public void cancelElection() throws InterruptedException, KeeperException {
     try {
+      log.info("canceling election {}",leaderSeqPath );
       zkClient.delete(leaderSeqPath, -1, true);
     } catch (NoNodeException e) {
       // fine

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1584271&r1=1584270&r2=1584271&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java Thu Apr  3 07:07:50 2014
@@ -283,6 +283,7 @@ public class Overseer {
 
         }
       } finally {
+        log.info("Overseer Loop exiting : {}", LeaderElector.getNodeName(myId));
         new Thread("OverseerExitThread"){
           //do this in a separate thread because any wait is interrupted in this main thread
           @Override
@@ -372,7 +373,7 @@ public class Overseer {
       } else if(CLUSTERPROP.isEqual(operation)){
            handleProp(message);
       } else if( QUIT.equals(operation)){
-        log.info("################Quit command receive");
+        log.info("Quit command received {}", LeaderElector.getNodeName(myId));
         overseerCollectionProcessor.close();
         close();
       } else{

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1584271&r1=1584270&r2=1584271&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Thu Apr  3 07:07:50 2014
@@ -289,7 +289,7 @@ public class OverseerCollectionProcessor
   }
 
   private void prioritizeOverseerNodes() throws KeeperException, InterruptedException {
-    log.info("prioritizing overseer nodes");
+    log.info("prioritizing overseer nodes at {}", LeaderElector.getNodeName(myId));
     SolrZkClient zk = zkStateReader.getZkClient();
     if(!zk.exists(ZkStateReader.ROLES,true))return;
     Map m = (Map) ZkStateReader.fromJSON(zk.getData(ZkStateReader.ROLES, null, new Stat(), true));
@@ -301,6 +301,7 @@ public class OverseerCollectionProcessor
 
     List<String> nodeNames = getSortedOverseerNodeNames(zk);
     if(nodeNames.size()<2) return;
+    boolean designateIsInFront = overseerDesignates.contains( nodeNames.get(0));
 
 //
     ArrayList<String> nodesTobePushedBack =  new ArrayList<>();
@@ -322,7 +323,7 @@ public class OverseerCollectionProcessor
         }
 
       }
-      if(availableDesignates.size()>1) break;
+      if(availableDesignates.size()>1) break;//we don't need to line up more than 2 designates
     }
 
     if(!availableDesignates.isEmpty()){
@@ -360,17 +361,22 @@ public class OverseerCollectionProcessor
         log.warn("available designates and current state {} {} ", availableDesignates, getSortedOverseerNodeNames(zk));
       }
 
-    } else {
+    } else if(!designateIsInFront) {
       log.warn("No overseer designates are available, overseerDesignates: {}, live nodes : {}",overseerDesignates,nodeNames);
       return;
     }
 
     String leaderNode = getLeaderNode(zkStateReader.getZkClient());
     if(leaderNode ==null) return;
-    if(!overseerDesignates.contains(leaderNode) && !availableDesignates.isEmpty()){
-      //this means there are designated Overseer nodes and I am not one of them , kill myself
-      log.info("I am not an overseer designate , forcing myself out {} ", leaderNode);
-      Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(new ZkNodeProps( Overseer.QUEUE_OPERATION, Overseer.QUIT)));
+    if(!overseerDesignates.contains(leaderNode) ){
+      List<String> sortedNodes = getSortedOverseerNodeNames(zk);
+
+      if(leaderNode.equals(sortedNodes.get(0))  ||         // I am leader and I am in front of the queue
+          overseerDesignates.contains(sortedNodes.get(0))) {// I am leader but somebody else is in the front , Screwed up leader election
+        //this means there are I am not a designate and the next guy is lined up to become the leader, kill myself
+        log.info("I am not an overseer designate , forcing myself out {} ", leaderNode);
+        Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, Overseer.QUIT)));
+      }
     }
 
   }