You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/02/17 00:29:24 UTC

svn commit: r1568857 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java

Author: markrmiller
Date: Sun Feb 16 23:29:23 2014
New Revision: 1568857

URL: http://svn.apache.org/r1568857
Log:
 SOLR-5727: LBHttpSolrServer should only retry on Connection exceptions when sending updates. Affects CloudSolrServer. 

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1568857&r1=1568856&r2=1568857&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sun Feb 16 23:29:23 2014
@@ -337,6 +337,9 @@ Bug Fixes
 * SOLR-5718: Make LBHttpSolrServer zombie checks non-distrib and non-scoring.
   (Christine Poerschke via Mark Miller)
 
+* SOLR-5727: LBHttpSolrServer should only retry on Connection exceptions when 
+  sending updates. Affects CloudSolrServer. (Mark Miller)
+
 Optimizations
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1568857&r1=1568856&r2=1568857&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Sun Feb 16 23:29:23 2014
@@ -204,7 +204,7 @@ public class ChaosMonkeyNothingIsSafeTes
       // we expect full throttle fails, but cloud client should not easily fail
       for (StopableThread indexThread : threads) {
         if (indexThread instanceof StopableIndexingThread && !(indexThread instanceof FullThrottleStopableIndexingThread)) {
-          assertFalse("There were too many update fails - we expect it can happen, but shouldn't easily", ((StopableIndexingThread) indexThread).getFails() > 1);
+          assertFalse("There were too many update fails - we expect it can happen, but shouldn't easily", ((StopableIndexingThread) indexThread).getFails() > 10);
         }
       }
       

Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java?rev=1568857&r1=1568856&r2=1568857&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java Sun Feb 16 23:29:23 2014
@@ -18,6 +18,7 @@ package org.apache.solr.client.solrj.imp
 
 import org.apache.http.client.HttpClient;
 import org.apache.solr.client.solrj.*;
+import org.apache.solr.client.solrj.request.IsUpdateRequest;
 import org.apache.solr.client.solrj.request.RequestWriter;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -27,6 +28,7 @@ import org.apache.solr.common.SolrExcept
 
 import java.io.IOException;
 import java.lang.ref.WeakReference;
+import java.net.ConnectException;
 import java.net.MalformedURLException;
 import java.net.SocketException;
 import java.net.SocketTimeoutException;
@@ -74,7 +76,14 @@ import java.util.*;
  * @since solr 1.4
  */
 public class LBHttpSolrServer extends SolrServer {
+  private static Set<Integer> RETRY_CODES = new HashSet<Integer>(4);
 
+  static {
+    RETRY_CODES.add(404);
+    RETRY_CODES.add(403);
+    RETRY_CODES.add(503);
+    RETRY_CODES.add(500);
+  }
 
   // keys to the maps are currently of the form "http://localhost:8983/solr"
   // which should be equivalent to CommonsHttpSolrServer.getBaseURL()
@@ -273,7 +282,7 @@ public class LBHttpSolrServer extends So
   public Rsp request(Req req) throws SolrServerException, IOException {
     Rsp rsp = new Rsp();
     Exception ex = null;
-
+    boolean isUpdate = req.request instanceof IsUpdateRequest;
     List<ServerWrapper> skipped = new ArrayList<ServerWrapper>(req.getNumDeadServersToTry());
 
     for (String serverStr : req.getServers()) {
@@ -293,25 +302,31 @@ public class LBHttpSolrServer extends So
         rsp.rsp = server.request(req.getRequest());
         return rsp; // SUCCESS
       } catch (SolrException e) {
-        // we retry on 404 or 403 or 503 - you can see this on solr shutdown
-        if (e.code() == 404 || e.code() == 403 || e.code() == 503 || e.code() == 500) {
+        // we retry on 404 or 403 or 503 or 500
+        // unless it's an update - then we only retry on connect exceptions
+        if (!isUpdate && RETRY_CODES.contains(e.code())) {
           ex = addZombie(server, e);
         } else {
           // Server is alive but the request was likely malformed or invalid
           throw e;
         }
-       
-       // TODO: consider using below above - currently does cause a problem with distrib updates:
-       // seems to match up against a failed forward to leader exception as well...
-       //     || e.getMessage().contains("java.net.SocketException")
-       //     || e.getMessage().contains("java.net.ConnectException")
       } catch (SocketException e) {
-        ex = addZombie(server, e);
+        if (!isUpdate || e instanceof ConnectException) {
+          ex = addZombie(server, e);
+        } else {
+          throw e;
+        }
       } catch (SocketTimeoutException e) {
-        ex = addZombie(server, e);
+        if (!isUpdate) {
+          ex = addZombie(server, e);
+        } else {
+          throw e;
+        }
       } catch (SolrServerException e) {
         Throwable rootCause = e.getRootCause();
-        if (rootCause instanceof IOException) {
+        if (!isUpdate && rootCause instanceof IOException) {
+          ex = addZombie(server, e);
+        } else if (isUpdate && rootCause instanceof ConnectException) {
           ex = addZombie(server, e);
         } else {
           throw e;
@@ -328,8 +343,9 @@ public class LBHttpSolrServer extends So
         zombieServers.remove(wrapper.getKey());
         return rsp; // SUCCESS
       } catch (SolrException e) {
-        // we retry on 404 or 403 or 503 - you can see this on solr shutdown
-        if (e.code() == 404 || e.code() == 403 || e.code() == 503 || e.code() == 500) {
+        // we retry on 404 or 403 or 503 or 500
+        // unless it's an update - then we only retry on connect exceptions
+        if (!isUpdate && RETRY_CODES.contains(e.code())) {
           ex = e;
           // already a zombie, no need to re-add
         } else {
@@ -339,14 +355,23 @@ public class LBHttpSolrServer extends So
         }
 
       } catch (SocketException e) {
-        ex = e;
+        if (!isUpdate || e instanceof ConnectException) {
+          ex = e;
+        } else {
+          throw e;
+        }
       } catch (SocketTimeoutException e) {
-        ex = e;
+        if (!isUpdate) {
+          ex = e;
+        } else {
+          throw e;
+        }
       } catch (SolrServerException e) {
         Throwable rootCause = e.getRootCause();
-        if (rootCause instanceof IOException) {
+        if (!isUpdate && rootCause instanceof IOException) {
+          ex = e;
+        } else if (isUpdate && rootCause instanceof ConnectException) {
           ex = e;
-          // already a zombie, no need to re-add
         } else {
           throw e;
         }