You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/09 03:47:25 UTC
svn commit: r1229009 - in /lucene/dev/branches/solrcloud/solr:
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/update/processor/
core/src/test/org/apache/solr/cloud/
solrj/src/java/org/apache/solr/common/cloud/
Author: markrmiller
Date: Mon Jan 9 02:47:25 2012
New Revision: 1229009
URL: http://svn.apache.org/viewvc?rev=1229009&view=rev
Log:
some test and other minor changes
Modified:
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java
lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Mon Jan 9 02:47:25 2012
@@ -30,11 +30,9 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZooKeeperException;
-import org.apache.solr.common.cloud.ZkOperation;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.ConnectionLossException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.slf4j.Logger;
@@ -127,6 +125,8 @@ public class LeaderElector {
}
}, null, true);
+ } catch (KeeperException.SessionExpiredException e) {
+ throw e;
} catch (KeeperException e) {
// we couldn't set our watch - the node before us may already be down?
// we need to check if we are the leader again
Modified: lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Mon Jan 9 02:47:25 2012
@@ -72,7 +72,7 @@ public class DistributedUpdateProcessor
private final SolrQueryResponse rsp;
private final UpdateRequestProcessor next;
- private static final String VERSION_FIELD = "_version_";
+ public static final String VERSION_FIELD = "_version_";
private final UpdateHandler updateHandler;
private final UpdateLog ulog;
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/BasicSolrCloudTest.java Mon Jan 9 02:47:25 2012
@@ -26,8 +26,11 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.update.VersionInfo;
+import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.junit.BeforeClass;
/**
@@ -162,6 +165,8 @@ public class BasicSolrCloudTest extends
assertEquals(2, results.getResults().getNumFound());
testIndexingWithSuss();
+
+ // TODO: testOptimisticUpdate(results);
}
private void testIndexingWithSuss() throws MalformedURLException, Exception {
@@ -177,6 +182,49 @@ public class BasicSolrCloudTest extends
checkShardConsistency(false);
}
+
+ private void testOptimisticUpdate(QueryResponse results) throws Exception {
+ SolrDocument doc = results.getResults().get(0);
+ System.out.println("version:" + doc.getFieldValue(VersionInfo.VERSION_FIELD));
+ Long version = (Long) doc.getFieldValue(VersionInfo.VERSION_FIELD);
+ Integer theDoc = (Integer) doc.getFieldValue("id");
+ UpdateRequest uReq = new UpdateRequest();
+ SolrInputDocument doc1 = new SolrInputDocument();
+ uReq.setParams(new ModifiableSolrParams());
+ uReq.getParams().set(DistributedUpdateProcessor.VERSION_FIELD, Long.toString(version));
+ addFields(doc1, "id", theDoc, t1, "theupdatestuff");
+ uReq.add(doc1);
+
+ uReq.process(cloudClient);
+ uReq.process(controlClient);
+
+ commit();
+
+ // updating the old version should fail...
+ SolrInputDocument doc2 = new SolrInputDocument();
+ uReq = new UpdateRequest();
+ uReq.setParams(new ModifiableSolrParams());
+ uReq.getParams().set(DistributedUpdateProcessor.VERSION_FIELD, Long.toString(version));
+ addFields(doc2, "id", theDoc, t1, "thenewupdatestuff");
+ uReq.add(doc2);
+
+ uReq.process(cloudClient);
+ uReq.process(controlClient);
+
+ commit();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("distrib", "true");
+ params.add("q", t1 + ":thenewupdatestuff");
+ QueryResponse res = clients.get(0).query(params);
+ assertEquals(0, res.getResults().getNumFound());
+
+ params = new ModifiableSolrParams();
+ params.add("distrib", "true");
+ params.add("q", t1 + ":theupdatestuff");
+ res = clients.get(0).query(params);
+ assertEquals(1, res.getResults().getNumFound());
+ }
private QueryResponse query(SolrServer server) throws SolrServerException {
SolrQuery query = new SolrQuery("*:*");
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Mon Jan 9 02:47:25 2012
@@ -992,7 +992,9 @@ public class FullSolrCloudTest extends A
// wait for recovery to finish
// if it takes over n seconds, assume we didnt get our listener attached before
// recover started - it should be done before n though
- recoveryLatch.await(30, TimeUnit.SECONDS);
+ if (!recoveryLatch.await(45, TimeUnit.SECONDS)) {
+ log.warn("Timed out waiting to be notified of replication");
+ }
}
@Override
Modified: lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java (original)
+++ lucene/dev/branches/solrcloud/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java Mon Jan 9 02:47:25 2012
@@ -72,7 +72,7 @@ public class LeaderElectionTest extends
AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
- seqToThread = new HashMap<Integer,Thread>();
+ seqToThread = Collections.synchronizedMap(new HashMap<Integer,Thread>());
}
class ClientThread extends Thread {
@@ -234,18 +234,25 @@ public class LeaderElectionTest extends
@Test
public void testStressElection() throws Exception {
- //TODO add assertions
final ScheduledExecutorService scheduler = Executors
- .newScheduledThreadPool(100);
+ .newScheduledThreadPool(15);
final List<ClientThread> threads = Collections
.synchronizedList(new ArrayList<ClientThread>());
+ // start with a leader
+ ClientThread thread1 = null;
+ thread1 = new ClientThread(0);
+ threads.add(thread1);
+ scheduler.schedule(thread1, 0, TimeUnit.MILLISECONDS);
+
+ Thread.sleep(4000);
+
Thread scheduleThread = new Thread() {
@Override
public void run() {
- for (int i = 0; i < 300; i++) {
- int launchIn = random.nextInt(6000);
+ for (int i = 1; i < atLeast(15); i++) {
+ int launchIn = random.nextInt(500);
ClientThread thread = null;
try {
thread = new ClientThread(i);
@@ -260,8 +267,6 @@ public class LeaderElectionTest extends
}
};
- scheduleThread.start();
-
Thread killThread = new Thread() {
@Override
public void run() {
@@ -270,7 +275,8 @@ public class LeaderElectionTest extends
try {
int j;
try {
- j = random.nextInt(threads.size());
+ // always 1 we won't kill...
+ j = random.nextInt(threads.size() - 1);
} catch(IllegalArgumentException e) {
continue;
}
@@ -296,48 +302,46 @@ public class LeaderElectionTest extends
public void run() {
while (!stopStress) {
+
try {
+ Thread.sleep(50);
int j;
+ j = random.nextInt(threads.size());
try {
- j = random.nextInt(threads.size());
- } catch(IllegalArgumentException e) {
- continue;
- }
- try {
- threads.get(j).zkClient.getSolrZooKeeper().pauseCnxn(ZkTestServer.TICK_TIME * 2);
+ threads.get(j).zkClient.getSolrZooKeeper().pauseCnxn(
+ ZkTestServer.TICK_TIME * 2);
} catch (Exception e) {
e.printStackTrace();
}
- Thread.sleep(10);
+ Thread.sleep(500);
} catch (Exception e) {
-
+
}
}
}
};
+ scheduleThread.start();
connLossThread.start();
killThread.start();
- Thread.sleep(10000);
+ Thread.sleep(6000);
+
+ stopStress = true;
scheduleThread.interrupt();
connLossThread.interrupt();
killThread.interrupt();
- stopStress = true;
-
scheduleThread.join();
connLossThread.join();
killThread.join();
- Thread.sleep(1000);
-
scheduler.shutdownNow();
- //printLayout(server.getZkAddress());
+ printLayout(server.getZkAddress());
System.out.println("leader thread:" + getLeaderThread());
@@ -346,7 +350,6 @@ public class LeaderElectionTest extends
System.out.println("Node:" + threads.get(getLeaderThread()).getNodeNumber());
assertFalse("seq is -1 and we may have a zombie leader", seq == -1);
-
// cleanup any threads still running
for (ClientThread thread : threads) {
Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java (original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java Mon Jan 9 02:47:25 2012
@@ -439,6 +439,9 @@ public class SolrZkClient {
* e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
* group, node exist, each will be created.
*
+ * Note: retryOnConnLoss is only respected for the final node - nodes
+ * before that are always retried on connection loss.
+ *
* @param path
* @param data
* @param createMode
@@ -453,6 +456,7 @@ public class SolrZkClient {
if (log.isInfoEnabled()) {
log.info("makePath: " + path);
}
+ boolean retry = true;
if (path.startsWith("/")) {
path = path.substring(1, path.length());
@@ -470,9 +474,11 @@ public class SolrZkClient {
if (i == paths.length - 1) {
mode = createMode;
bytes = data;
+ System.out.println("bytes = data");
+ if (!retryOnConnLoss) retry = false;
}
try {
- if (retryOnConnLoss) {
+ if (retry) {
final CreateMode finalMode = mode;
final byte[] finalBytes = bytes;
zkCmdExecutor.retryOperation(new ZkOperation() {
Modified: lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java?rev=1229009&r1=1229008&r2=1229009&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java (original)
+++ lucene/dev/branches/solrcloud/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java Mon Jan 9 02:47:25 2012
@@ -101,31 +101,22 @@ public class ZkCmdExecutor {
throw exception;
}
- public void ensureExists(String path, final SolrZkClient zkClient) {
+ public void ensureExists(String path, final SolrZkClient zkClient) throws KeeperException, InterruptedException {
ensureExists(path, null, CreateMode.PERSISTENT, zkClient);
}
public void ensureExists(final String path, final byte[] data,
- CreateMode createMode, final SolrZkClient zkClient) {
+ CreateMode createMode, final SolrZkClient zkClient) throws KeeperException, InterruptedException {
+
+ if (zkClient.exists(path, false)) {
+ return;
+ }
try {
- retryOperation(new ZkOperation() {
- public Object execute() throws KeeperException, InterruptedException {
- if (zkClient.exists(path, false)) {
- return true;
- }
- try {
- zkClient.makePath(path, data, false);
- } catch (NodeExistsException e) {
- // its okay if another beats us creating the node
- }
- return true;
- }
- });
- } catch (KeeperException e) {
- LOG.warn("", e);
- } catch (InterruptedException e) {
- LOG.warn("", e);
+ zkClient.makePath(path, data, true);
+ } catch (NodeExistsException e) {
+ // its okay if another beats us creating the node
}
+
}
/**