You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2015/11/04 18:59:38 UTC

svn commit: r1712601 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/servlet/ core/src/test/org/apache/solr/cloud/ test-framework/src/java/org/apache/solr/cloud/

Author: varun
Date: Wed Nov  4 17:59:37 2015
New Revision: 1712601

URL: http://svn.apache.org/viewvc?rev=1712601&view=rev
Log:
SOLR-8215: Only active replicas should handle incoming requests against a collection

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java
    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1712601&r1=1712600&r2=1712601&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Nov  4 17:59:37 2015
@@ -322,6 +322,8 @@ Bug Fixes
   (Mark Miller, yonik)
 
 
+* SOLR-8215: Only active replicas should handle incoming requests against a collection (Varun Thacker)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java?rev=1712601&r1=1712600&r2=1712601&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java Wed Nov  4 17:59:37 2015
@@ -765,26 +765,31 @@ public class HttpSolrCall {
     return result;
   }
 
-  private SolrCore getCoreByCollection(String corename) {
+  private SolrCore getCoreByCollection(String collection) {
     ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
 
     ClusterState clusterState = zkStateReader.getClusterState();
-    Map<String, Slice> slices = clusterState.getActiveSlicesMap(corename);
+    Map<String, Slice> slices = clusterState.getActiveSlicesMap(collection);
     if (slices == null) {
       return null;
     }
+    Set<String> liveNodes = clusterState.getLiveNodes();
     // look for a core on this node
     Set<Map.Entry<String, Slice>> entries = slices.entrySet();
     SolrCore core = null;
-    done:
+
+    //Hitting the leaders is useful when it's an update request.
+    //For queries it doesn't matter and hence we don't distinguish here.
     for (Map.Entry<String, Slice> entry : entries) {
       // first see if we have the leader
-      ZkNodeProps leaderProps = clusterState.getLeader(corename, entry.getKey());
-      if (leaderProps != null) {
-        core = checkProps(leaderProps);
-      }
-      if (core != null) {
-        break done;
+      Replica leaderProps = clusterState.getLeader(collection, entry.getKey());
+      if (liveNodes.contains(leaderProps.getNodeName()) && leaderProps.getState() == Replica.State.ACTIVE) {
+        if (leaderProps != null) {
+          core = checkProps(leaderProps);
+        }
+        if (core != null) {
+          return core;
+        }
       }
 
       // check everyone then
@@ -792,13 +797,15 @@ public class HttpSolrCall {
       Set<Map.Entry<String, Replica>> shardEntries = shards.entrySet();
       for (Map.Entry<String, Replica> shardEntry : shardEntries) {
         Replica zkProps = shardEntry.getValue();
-        core = checkProps(zkProps);
-        if (core != null) {
-          break done;
+        if (liveNodes.contains(zkProps.getNodeName()) && zkProps.getState() == Replica.State.ACTIVE) {
+          core = checkProps(zkProps);
+          if (core != null) {
+            return core;
+          }
         }
       }
     }
-    return core;
+    return null;
   }
 
   private SolrCore checkProps(ZkNodeProps zkProps) {

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1712601&r1=1712600&r2=1712601&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Wed Nov  4 17:59:37 2015
@@ -65,6 +65,8 @@ import org.junit.Ignore;
 import org.junit.Test;
 import org.xml.sax.SAXException;
 
+import static org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus;
+
 @Slow
 public class OverseerTest extends SolrTestCaseJ4 {
 
@@ -325,7 +327,7 @@ public class OverseerTest extends SolrTe
       
       // make sure the Overseer is still processing items
       for (int i = 0; i < numShards; i++) {
-        assertNotNull("shard got no id?", zkController.publishState("collection2", "core" + (i+1), "node" + (i+1), Replica.State.ACTIVE, 3));
+        assertNotNull("shard got no id?", zkController.publishState("collection2", "core" + (i + 1), "node" + (i + 1), Replica.State.ACTIVE, 3));
       }
 
       assertEquals(1, reader.getClusterState().getSlice("collection2", "shard1").getReplicasMap().size());
@@ -573,7 +575,7 @@ public class OverseerTest extends SolrTe
 
       q.offer(Utils.toJSON(m));
 
-      verifyStatus(reader, "collection1", "shard1", "core_node1", Replica.State.ACTIVE);
+      verifyReplicaStatus(reader, "collection1", "shard1", "core_node1", Replica.State.ACTIVE);
 
     } finally {
 
@@ -584,22 +586,6 @@ public class OverseerTest extends SolrTe
       server.shutdown();
     }
   }
-
-  private void verifyStatus(ZkStateReader reader, String collection, String shard, String coreNodeName, Replica.State expectedState) throws InterruptedException {
-    int maxIterations = 100;
-    Replica.State coreState = null;
-    while(maxIterations-->0) {
-      Slice slice = reader.getClusterState().getSlice(collection, shard);
-      if(slice!=null) {
-        coreState = slice.getReplicasMap().get(coreNodeName).getState();
-        if(coreState == expectedState) {
-          return;
-        }
-      }
-      Thread.sleep(50);
-    }
-    fail("Illegal state, was: " + coreState + " expected:" + expectedState + " clusterState:" + reader.getClusterState());
-  }
   
   private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException {
     int maxIterations = 200;
@@ -649,15 +635,15 @@ public class OverseerTest extends SolrTe
       mockController.publishState(collection, "core1", "core_node1", Replica.State.RECOVERING, 1);
       
       waitForCollections(reader, collection);
-      verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING);
+      verifyReplicaStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING);
       
       int version = getClusterStateVersion(zkClient);
       
       mockController.publishState(collection, "core1", "core_node1", Replica.State.ACTIVE, 1);
       
       while (version == getClusterStateVersion(zkClient));
-      
-      verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.ACTIVE);
+
+      verifyReplicaStatus(reader, collection, "shard1", "core_node1", Replica.State.ACTIVE);
       version = getClusterStateVersion(zkClient);
       overseerClient.close();
       Thread.sleep(1000); // wait for overseer to get killed
@@ -668,8 +654,8 @@ public class OverseerTest extends SolrTe
       overseerClient = electNewOverseer(server.getZkAddress());
       
       while (version == getClusterStateVersion(zkClient));
-      
-      verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING);
+
+      verifyReplicaStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING);
       
       assertEquals("Live nodes count does not match", 1, reader
           .getClusterState().getLiveNodes().size());
@@ -883,8 +869,8 @@ public class OverseerTest extends SolrTe
       mockController.publishState(collection, "core1", "core_node1", Replica.State.RECOVERING, 1);
 
       waitForCollections(reader, "collection1");
-      
-      verifyStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING);
+
+      verifyReplicaStatus(reader, collection, "shard1", "core_node1", Replica.State.RECOVERING);
 
       mockController.close();
 
@@ -1201,7 +1187,7 @@ public class OverseerTest extends SolrTe
       q.offer(Utils.toJSON(m));
 
       waitForCollections(reader, "c1");
-      verifyStatus(reader, "c1", "shard1", "core_node1", Replica.State.DOWN);
+      verifyReplicaStatus(reader, "c1", "shard1", "core_node1", Replica.State.DOWN);
 
       m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(),
           ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr",
@@ -1255,10 +1241,10 @@ public class OverseerTest extends SolrTe
       q.offer(Utils.toJSON(m));
 
       waitForCollections(reader, "test");
-      verifyStatus(reader, "test", "x", "core_node1", Replica.State.DOWN);
+      verifyReplicaStatus(reader, "test", "x", "core_node1", Replica.State.DOWN);
 
       waitForCollections(reader, "c1");
-      verifyStatus(reader, "c1", "shard1", "core_node1", Replica.State.ACTIVE);
+      verifyReplicaStatus(reader, "c1", "shard1", "core_node1", Replica.State.ACTIVE);
 
     } finally {
       close(zkClient);

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java?rev=1712601&r1=1712600&r2=1712601&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestRandomRequestDistribution.java Wed Nov  4 17:59:37 2015
@@ -29,36 +29,45 @@ import org.apache.solr.client.solrj.Solr
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.cloud.overseer.OverseerAction;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.component.ShardHandlerFactory;
-import org.apache.solr.handler.component.TrackingShardHandlerFactory;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.servlet.SolrDispatchFilter;
 import org.junit.Test;
 
-/**
- * Asserts that requests aren't always sent to the same poor node. See SOLR-7493
- */
+
 @SolrTestCaseJ4.SuppressSSL
 public class TestRandomRequestDistribution extends AbstractFullDistribZkTestBase {
 
+  List<String> nodeNames = new ArrayList<>(3);
+
   @Test
   @BaseDistributedSearchTestCase.ShardsFixed(num = 3)
-  public void testRequestTracking() throws Exception {
+  public void test() throws Exception {
     waitForThingsToLevelOut(30);
 
-    List<String> nodeNames = new ArrayList<>(3);
     for (CloudJettyRunner cloudJetty : cloudJettys) {
       nodeNames.add(cloudJetty.nodeName);
     }
     assertEquals(3, nodeNames.size());
 
+    testRequestTracking();
+    testQueryAgainstDownReplica();
+  }
+
+  /**
+   * Asserts that requests aren't always sent to the same poor node. See SOLR-7493
+   */
+  private void testRequestTracking() throws Exception {
+
     new CollectionAdminRequest.Create()
         .setCollectionName("a1x2")
         .setNumShards(1)
@@ -109,4 +118,77 @@ public class TestRandomRequestDistributi
       assertTrue("Shard " + entry.getKey() + " received all 10 requests", entry.getValue() != 10);
     }
   }
+
+  /**
+   * Asserts that requests against a collection are only served by a 'active' local replica
+   */
+  private void testQueryAgainstDownReplica() throws Exception {
+
+    log.info("Creating collection 'football' with 1 shard and 2 replicas");
+    new CollectionAdminRequest.Create()
+        .setCollectionName("football")
+        .setNumShards(1)
+        .setReplicationFactor(2)
+        .setCreateNodeSet(nodeNames.get(0) + ',' + nodeNames.get(1))
+        .process(cloudClient);
+
+    waitForRecoveriesToFinish("football", true);
+
+    cloudClient.getZkStateReader().updateClusterState();
+
+    Replica leader = null;
+    Replica notLeader = null;
+
+    Collection<Replica> replicas = cloudClient.getZkStateReader().getClusterState().getSlice("football", "shard1").getReplicas();
+    for (Replica replica : replicas) {
+      if (replica.getStr(ZkStateReader.LEADER_PROP) != null) {
+        leader = replica;
+      } else {
+        notLeader = replica;
+      }
+    }
+
+    //Simulate a replica being in down state.
+    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(),
+        ZkStateReader.BASE_URL_PROP, notLeader.getStr(ZkStateReader.BASE_URL_PROP),
+        ZkStateReader.NODE_NAME_PROP, notLeader.getStr(ZkStateReader.NODE_NAME_PROP),
+        ZkStateReader.COLLECTION_PROP, "football",
+        ZkStateReader.SHARD_ID_PROP, "shard1",
+        ZkStateReader.CORE_NAME_PROP, notLeader.getStr(ZkStateReader.CORE_NAME_PROP),
+        ZkStateReader.ROLES_PROP, "",
+        ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
+
+    log.info("Forcing {} to go into 'down' state", notLeader.getStr(ZkStateReader.CORE_NAME_PROP));
+    DistributedQueue q = Overseer.getInQueue(cloudClient.getZkStateReader().getZkClient());
+    q.offer(Utils.toJSON(m));
+
+    verifyReplicaStatus(cloudClient.getZkStateReader(), "football", "shard1", notLeader.getName(), Replica.State.DOWN);
+
+    //Query against the node which hosts the down replica
+
+    String baseUrl = notLeader.getStr(ZkStateReader.BASE_URL_PROP);
+    if (!baseUrl.endsWith("/")) baseUrl += "/";
+    String path = baseUrl + "football";
+    log.info("Firing query against path=" + path);
+    HttpSolrClient client = new HttpSolrClient(path);
+    client.setSoTimeout(5000);
+    client.setConnectionTimeout(2000);
+
+    client.query(new SolrQuery("*:*"));
+    client.close();
+
+    //Test to see if the query got forwarded to the active replica or not.
+    for (JettySolrRunner jetty : jettys) {
+      CoreContainer container = ((SolrDispatchFilter) jetty.getDispatchFilter().getFilter()).getCores();
+      for (SolrCore core : container.getCores()) {
+        if (core.getName().equals(leader.getStr(ZkStateReader.CORE_NAME_PROP))) {
+          SolrRequestHandler select = core.getRequestHandler("");
+          long c = (long) select.getStatistics().get("requests");
+          assertEquals(core.getName() + " should have got 1 request", 1, c);
+          break;
+        }
+      }
+    }
+
+  }
 }

Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java?rev=1712601&r1=1712600&r2=1712601&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java Wed Nov  4 17:59:37 2015
@@ -214,6 +214,22 @@ public abstract class AbstractDistribZkT
 
     log.info("Collection has disappeared - collection: " + collection);
   }
+
+  public static void verifyReplicaStatus(ZkStateReader reader, String collection, String shard, String coreNodeName, Replica.State expectedState) throws InterruptedException {
+    int maxIterations = 100;
+    Replica.State coreState = null;
+    while(maxIterations-->0) {
+      Slice slice = reader.getClusterState().getSlice(collection, shard);
+      if(slice!=null) {
+        coreState = slice.getReplicasMap().get(coreNodeName).getState();
+        if(coreState == expectedState) {
+          return;
+        }
+      }
+      Thread.sleep(50);
+    }
+    fail("Illegal state, was: " + coreState + " expected:" + expectedState + " clusterState:" + reader.getClusterState());
+  }
   
   protected void assertAllActive(String collection,ZkStateReader zkStateReader)
       throws KeeperException, InterruptedException {