You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/11/12 16:31:10 UTC
svn commit: r1408323 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/codecs/ lucene/core/
lucene/core/src/test/org/apache/l...
Author: markrmiller
Date: Mon Nov 12 15:31:06 2012
New Revision: 1408323
URL: http://svn.apache.org/viewvc?rev=1408323&view=rev
Log:
SOLR-3993: If multiple SolrCore's for a shard coexist on a node, on cluster restart, leader election would stall until timeout, waiting to see all of the replicas come up.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1408323&r1=1408322&r2=1408323&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Mon Nov 12 15:31:06 2012
@@ -148,6 +148,10 @@ Bug Fixes
cause actions to be targeted at the wrong SolrCores.
(Raintung Li via Mark Miller)
+* SOLR-3993: If multiple SolrCore's for a shard coexist on a node, on cluster
+ restart, leader election would stall until timeout, waiting to see all of
+ the replicas come up. (Mark Miller, Alexey Kudinov)
+
Other Changes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml?rev=1408323&r1=1408322&r2=1408323&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml Mon Nov 12 15:31:06 2012
@@ -22,7 +22,7 @@
persistent: Save changes made via the API to this file
sharedLib: path to a lib directory that will be shared across all cores
-->
-<solr persistent="false">
+<solr persistent="${solr.xml.persist:false}">
<!--
adminPath: RequestHandler path to manage cores.
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1408323&r1=1408322&r2=1408323&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Mon Nov 12 15:31:06 2012
@@ -119,6 +119,7 @@ public class BasicDistributedZkTest exte
public void setUp() throws Exception {
super.setUp();
System.setProperty("numShards", Integer.toString(sliceCount));
+ System.setProperty("solr.xml.persist", "true");
}
@@ -334,6 +335,7 @@ public class BasicDistributedZkTest exte
testCollectionsAPI();
testCoreUnloadAndLeaders();
testUnloadLotsOfCores();
+ testStopAndStartCoresInOneInstance();
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
@@ -541,54 +543,97 @@ public class BasicDistributedZkTest exte
5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
int cnt = atLeast(6);
- for (int i = 0; i < cnt; i++) {
- final int freezeI = i;
+
+ // create the 6 cores
+ createCores(server, executor, "multiunload", 2, cnt);
+
+ executor.shutdown();
+ executor.awaitTermination(120, TimeUnit.SECONDS);
+ executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
+ TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ new DefaultSolrThreadFactory("testExecutor"));
+ for (int j = 0; j < cnt; j++) {
+ final int freezeJ = j;
executor.execute(new Runnable() {
-
@Override
public void run() {
- Create createCmd = new Create();
- createCmd.setCoreName("multiunload" + freezeI);
- createCmd.setCollection("multiunload");
- String core3dataDir = dataDir.getAbsolutePath() + File.separator
- + System.currentTimeMillis() + "unloadcollection" + "_3n" + freezeI;
- createCmd.setDataDir(core3dataDir);
+ Unload unloadCmd = new Unload(true);
+ unloadCmd.setCoreName("multiunload" + freezeJ);
try {
- server.request(createCmd);
+ server.request(unloadCmd);
} catch (SolrServerException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
-
});
+ Thread.sleep(random().nextInt(50));
}
executor.shutdown();
executor.awaitTermination(120, TimeUnit.SECONDS);
- executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
- TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ }
+
+ private void testStopAndStartCoresInOneInstance() throws Exception {
+ SolrServer client = clients.get(0);
+ String url3 = getBaseUrl(client);
+ final HttpSolrServer server = new HttpSolrServer(url3);
+
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
+ 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
new DefaultSolrThreadFactory("testExecutor"));
- for (int j = 0; j < cnt; j++) {
- final int freezeJ = j;
+ int cnt = 3;
+
+ // create the cores
+ createCores(server, executor, "multiunload2", 1, cnt);
+
+ executor.shutdown();
+ executor.awaitTermination(120, TimeUnit.SECONDS);
+
+ ChaosMonkey.stop(cloudJettys.get(0).jetty);
+ printLayout();
+ // nocommit
+ System.out.println("start again");
+ Thread.sleep(5000);
+ ChaosMonkey.start(cloudJettys.get(0).jetty);
+ cloudClient.getZkStateReader().updateClusterState(true);
+ try {
+ cloudClient.getZkStateReader().getLeaderProps("multiunload2", "shard1", 30000);
+ } catch (SolrException e) {
+ printLayout();
+ throw e;
+ }
+
+ printLayout();
+
+ }
+
+ private void createCores(final HttpSolrServer server,
+ ThreadPoolExecutor executor, final String collection, final int numShards, int cnt) {
+ for (int i = 0; i < cnt; i++) {
+ final int freezeI = i;
executor.execute(new Runnable() {
+
@Override
public void run() {
- Unload unloadCmd = new Unload(true);
- unloadCmd.setCoreName("multiunload" + freezeJ);
+ Create createCmd = new Create();
+ createCmd.setCoreName(collection + freezeI);
+ createCmd.setCollection(collection);
+ String core3dataDir = dataDir.getAbsolutePath() + File.separator
+ + System.currentTimeMillis() + collection + "_3n" + freezeI;
+ createCmd.setDataDir(core3dataDir);
+ createCmd.setNumShards(numShards);
try {
- server.request(unloadCmd);
+ server.request(createCmd);
} catch (SolrServerException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
+
});
- Thread.sleep(random().nextInt(50));
}
- executor.shutdown();
- executor.awaitTermination(120, TimeUnit.SECONDS);
}
private String getBaseUrl(SolrServer client) {
@@ -1418,6 +1463,7 @@ public class BasicDistributedZkTest exte
}
System.clearProperty("numShards");
System.clearProperty("zkHost");
+ System.clearProperty("solr.xml.persist");
// insurance
DirectUpdateHandler2.commitOnClose = true;
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1408323&r1=1408322&r2=1408323&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Mon Nov 12 15:31:06 2012
@@ -403,7 +403,7 @@ public class ZkStateReader {
long timeoutAt = System.currentTimeMillis() + timeout;
while (System.currentTimeMillis() < timeoutAt) {
if (clusterState != null) {
- final ZkNodeProps nodeProps = clusterState.getLeader(collection, shard);
+ final ZkNodeProps nodeProps = clusterState.getLeader(collection, shard);
if (nodeProps != null && getClusterState().liveNodesContain((String) nodeProps.get(ZkStateReader.NODE_NAME_PROP))) {
return nodeProps;
}
Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1408323&r1=1408322&r2=1408323&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Mon Nov 12 15:31:06 2012
@@ -340,8 +340,12 @@ public abstract class AbstractFullDistri
return jettys;
}
- protected int getNumShards(String defaultCollection) {
- Map<String,Slice> slices = this.zkStateReader.getClusterState().getSlices(defaultCollection);
+ protected int getNumShards(String collection) {
+ ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(collection);
+ if (slices == null) {
+ throw new IllegalArgumentException("Could not find collection:" + collection);
+ }
int cnt = 0;
for (Map.Entry<String,Slice> entry : slices.entrySet()) {
cnt += entry.getValue().getReplicasMap().size();