You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/10 16:20:45 UTC
svn commit: r1229590 - in /lucene/dev/branches/solrcloud/solr:
core/src/java/org/apache/solr/update/processor/
core/src/test/org/apache/solr/cloud/
solrj/src/java/org/apache/solr/client/solrj/impl/
solrj/src/java/org/apache/solr/common/cloud/
Author: markrmiller
Date: Tue Jan 10 15:20:45 2012
New Revision: 1229590
URL: http://svn.apache.org/viewvc?rev=1229590&view=rev
Log:
fail when update sees a leader in the cluster state but cannot be forwarded to the leader - and test
Modified:
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java
lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1229590&r1=1229589&r2=1229590&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Tue Jan 10 15:20:45 2012
@@ -254,9 +254,13 @@ public class DistributedUpdateProcessor
// otherwise we assume things are fine if we got it locally
// until we start allowing min replication param
if (response.errors.size() > 0) {
+ // if one node is a RetryNode, this was a forward request
+ if (response.errors.get(0).node instanceof RetryNode) {
+ rsp.setException(response.errors.get(0).e);
+ }
+ // else
// for now we don't error - we assume if it was added locally, we
- // succeeded - nocommit: forwards should error
- //rsp.setException(response.errors.get(0).e);
+ // succeeded
}
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java?rev=1229590&r1=1229589&r2=1229590&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java Tue Jan 10 15:20:45 2012
@@ -17,7 +17,11 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import java.io.IOException;
import java.net.MalformedURLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
@@ -27,10 +31,16 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.update.VersionInfo;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
/**
@@ -60,63 +70,14 @@ public class BasicSolrCloudTest extends
// add a doc, update it, and delete it
- long docId = 99999999L;
- indexr("id", docId, t1, "originalcontent");
-
- commit();
-
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add("distrib", "true");
- params.add("q", t1 + ":originalcontent");
- QueryResponse results = clients.get(0).query(params);
- assertEquals(1, results.getResults().getNumFound());
-
- // update doc
- indexr("id", docId, t1, "updatedcontent");
-
- commit();
-
- assertDocCounts(VERBOSE);
-
- results = clients.get(0).query(params);
- assertEquals(0, results.getResults().getNumFound());
-
- params.set("q", t1 + ":updatedcontent");
-
- results = clients.get(0).query(params);
- assertEquals(1, results.getResults().getNumFound());
-
- UpdateRequest uReq = new UpdateRequest();
- //uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);
- uReq.deleteById(Long.toString(docId)).process(clients.get(0));
-
- commit();
-
- results = clients.get(0).query(params);
- assertEquals(0, results.getResults().getNumFound());
+ QueryResponse results;
+ UpdateRequest uReq;
+ long docId = addUpdateDelete();
// add 2 docs in a request
- uReq = new UpdateRequest();
- //uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);
- SolrInputDocument doc1 = new SolrInputDocument();
-
- addFields(doc1, "id", docId++);
- uReq.add(doc1);
- SolrInputDocument doc2 = new SolrInputDocument();
- addFields(doc2, "id", docId++);
- uReq.add(doc2);
-
- uReq.process(cloudClient);
- uReq.process(controlClient);
-
- commit();
-
- checkShardConsistency();
-
- assertDocCounts(VERBOSE);
-
- results = query(cloudClient);
- assertEquals(2, results.getResults().getNumFound());
+ SolrInputDocument doc1;
+ SolrInputDocument doc2;
+ docId = addTwoDocsInOneRequest(docId);
// two deletes
uReq = new UpdateRequest();
@@ -168,6 +129,108 @@ public class BasicSolrCloudTest extends
// TODO: testOptimisticUpdate(results);
+ testDeleteByQueryDistrib();
+
+ testThatCantForwardToLeaderFails();
+ }
+
+ private void testThatCantForwardToLeaderFails() throws InterruptedException,
+ Exception, TimeoutException, IOException, KeeperException {
+ ZkNodeProps props = zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard1");
+
+ chaosMonkey.stopShard("shard1");
+
+ // fake that the leader is still advertised
+ String leaderPath = ZkStateReader.getShardLeadersPath(DEFAULT_COLLECTION, "shard1");
+ SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), 10000);
+ int fails = 0;
+ try {
+ zkClient.makePath(leaderPath, ZkStateReader.toJSON(props),
+ CreateMode.EPHEMERAL, true);
+ for (int i = 200; i < 210; i++) {
+ try {
+ index_specific(cloudClient, id, i);
+ } catch (SolrException e) {
+ // expected
+ fails++;
+ break;
+ }
+ }
+ } finally {
+ zkClient.close();
+ }
+
+ assertTrue("A whole shard is down - some of these should fail", fails > 0);
+ }
+
+ private long addTwoDocsInOneRequest(long docId) throws SolrServerException,
+ IOException, Exception {
+ QueryResponse results;
+ UpdateRequest uReq;
+ uReq = new UpdateRequest();
+ //uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);
+ SolrInputDocument doc1 = new SolrInputDocument();
+
+ addFields(doc1, "id", docId++);
+ uReq.add(doc1);
+ SolrInputDocument doc2 = new SolrInputDocument();
+ addFields(doc2, "id", docId++);
+ uReq.add(doc2);
+
+ uReq.process(cloudClient);
+ uReq.process(controlClient);
+
+ commit();
+
+ checkShardConsistency();
+
+ assertDocCounts(VERBOSE);
+
+ results = query(cloudClient);
+ assertEquals(2, results.getResults().getNumFound());
+ return docId;
+ }
+
+ private long addUpdateDelete() throws Exception, SolrServerException,
+ IOException {
+ long docId = 99999999L;
+ indexr("id", docId, t1, "originalcontent");
+
+ commit();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("distrib", "true");
+ params.add("q", t1 + ":originalcontent");
+ QueryResponse results = clients.get(0).query(params);
+ assertEquals(1, results.getResults().getNumFound());
+
+ // update doc
+ indexr("id", docId, t1, "updatedcontent");
+
+ commit();
+
+ assertDocCounts(VERBOSE);
+
+ results = clients.get(0).query(params);
+ assertEquals(0, results.getResults().getNumFound());
+
+ params.set("q", t1 + ":updatedcontent");
+
+ results = clients.get(0).query(params);
+ assertEquals(1, results.getResults().getNumFound());
+
+ UpdateRequest uReq = new UpdateRequest();
+ //uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN);
+ uReq.deleteById(Long.toString(docId)).process(clients.get(0));
+
+ commit();
+
+ results = clients.get(0).query(params);
+ assertEquals(0, results.getResults().getNumFound());
+ return docId;
+ }
+
+ private void testDeleteByQueryDistrib() throws Exception, SolrServerException {
del("*:*");
commit();
assertEquals(0, query(cloudClient).getResults().getNumFound());
@@ -238,6 +301,7 @@ public class BasicSolrCloudTest extends
@Override
public void tearDown() throws Exception {
+ printLayout();
super.tearDown();
}
Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java?rev=1229590&r1=1229589&r2=1229590&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java (original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java Tue Jan 10 15:20:45 2012
@@ -27,9 +27,9 @@ import org.apache.solr.common.SolrExcept
import java.io.IOException;
import java.lang.ref.WeakReference;
-import java.net.ConnectException;
import java.net.MalformedURLException;
import java.net.SocketException;
+import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -252,7 +252,6 @@ public class LBHttpSolrServer extends So
return rsp; // SUCCESS
} catch (SolrException e) {
// we retry on 404 or 403 or 503 - you can see this on solr shutdown
- System.err.println("code:" + e.code());
if (e.code() == 404 || e.code() == 403 || e.code() == 503) {
ex = addZombie(server, e);
} else {
@@ -266,9 +265,11 @@ public class LBHttpSolrServer extends So
// || e.getMessage().contains("java.net.ConnectException")
} catch (SocketException e) {
ex = addZombie(server, e);
+ } catch (SocketTimeoutException e) {
+ ex = addZombie(server, e);
} catch (SolrServerException e) {
Throwable rootCause = e.getRootCause();
- if (rootCause instanceof IOException || rootCause instanceof ConnectException) {
+ if (rootCause instanceof IOException) {
ex = addZombie(server, e);
} else {
throw e;
@@ -295,8 +296,13 @@ public class LBHttpSolrServer extends So
throw e;
}
+ } catch (SocketException e) {
+ ex = e;
+ } catch (SocketTimeoutException e) {
+ ex = e;
} catch (SolrServerException e) {
- if (e.getRootCause() instanceof IOException) {
+ Throwable rootCause = e.getRootCause();
+ if (rootCause instanceof IOException) {
ex = e;
// already a zombie, no need to re-add
} else {
Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1229590&r1=1229589&r2=1229590&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Tue Jan 10 15:20:45 2012
@@ -378,7 +378,7 @@ public class ZkStateReader {
}
public ZkNodeProps getLeaderProps(String collection, String shard) throws InterruptedException {
- int tries = 120;
+ int tries = 50;
while (tries-- > 0) {
if (cloudState != null) {
final CloudState currentState = cloudState;
@@ -393,7 +393,7 @@ public class ZkStateReader {
}
}
}
- Thread.sleep(500);
+ Thread.sleep(50);
}
throw new RuntimeException("No registered leader was found, collection:" + collection + " slice:" + shard);
}