You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/02/10 03:24:25 UTC
svn commit: r1242639 - in /lucene/dev/trunk/solr/core/src:
java/org/apache/solr/cloud/Overseer.java
test/org/apache/solr/cloud/FullSolrCloudTest.java
test/org/apache/solr/cloud/OverseerTest.java
Author: markrmiller
Date: Fri Feb 10 02:24:25 2012
New Revision: 1242639
URL: http://svn.apache.org/viewvc?rev=1242639&view=rev
Log:
SOLR-3088: create shard placeholders
Modified:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1242639&r1=1242638&r2=1242639&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java Fri Feb 10 02:24:25 2012
@@ -71,7 +71,6 @@ public class Overseer implements NodeSta
}
}
- public static final String ASSIGNMENTS_NODE = "/node_assignments";
public static final String STATES_NODE = "/node_states";
private static Logger log = LoggerFactory.getLogger(Overseer.class);
@@ -189,7 +188,13 @@ public class Overseer implements NodeSta
private CloudState updateState(CloudState state, String nodeName, CoreState coreState) throws KeeperException, InterruptedException {
String collection = coreState.getCollectionName();
String zkCoreNodeName = coreState.getCoreNodeName();
-
+
+ //collection does not yet exist, create placeholders if num shards is specified
+ if (!state.getCollections().contains(coreState.getCollectionName())
+ && coreState.getNumShards() != null) {
+ state = createCollection(state, collection, coreState.getNumShards());
+ }
+
// use the provided non null shardId
String shardId = coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP);
if(shardId==null) {
@@ -220,6 +225,19 @@ public class Overseer implements NodeSta
return newCloudState;
}
+ private CloudState createCollection(CloudState state, String collectionName, int numShards) {
+ Map<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String, Slice>>();
+ Map<String, Slice> newSlices = new LinkedHashMap<String,Slice>();
+ newStates.putAll(state.getCollectionStates());
+ for (int i = 0; i < numShards; i++) {
+ final String sliceName = "shard" + (i+1);
+ newSlices.put(sliceName, new Slice(sliceName, Collections.EMPTY_MAP));
+ }
+ newStates.put(collectionName, newSlices);
+ CloudState newCloudState = new CloudState(state.getLiveNodes(), newStates);
+ return newCloudState;
+ }
+
/*
* Return an already assigned id or null if not assigned
*/
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1242639&r1=1242638&r2=1242639&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Fri Feb 10 02:24:25 2012
@@ -222,9 +222,14 @@ public class FullSolrCloudTest extends A
protected void createServers(int numServers) throws Exception {
System.setProperty("collection", "control_collection");
+ String numShards = System.getProperty(ZkStateReader.NUM_SHARDS_PROP);
+ System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
controlJetty = createJetty(testDir, testDir + "/control/data",
"control_shard");
System.clearProperty("collection");
+ if(numShards != null) {
+ System.setProperty(ZkStateReader.NUM_SHARDS_PROP, numShards);
+ }
controlClient = createNewSolrServer(controlJetty.getLocalPort());
createJettys(numServers, true);
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1242639&r1=1242638&r2=1242639&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Fri Feb 10 02:24:25 2012
@@ -156,8 +156,8 @@ public class OverseerTest extends SolrTe
assertEquals("shard2", ids[4]);
assertEquals("shard3", ids[5]);
- waitForSliceCount(reader, "collection1", 3);
-
+ waitForCollections(reader, "collection1");
+
//make sure leaders are in cloud state
assertNotNull(reader.getLeaderUrl("collection1", "shard1", 15000));
assertNotNull(reader.getLeaderUrl("collection1", "shard2", 15000));
@@ -345,20 +345,6 @@ public class OverseerTest extends SolrTe
System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
}
- //wait until i slices for collection have appeared
- private void waitForSliceCount(ZkStateReader stateReader, String collection, int i) throws InterruptedException, KeeperException {
- waitForCollections(stateReader, collection);
- int maxIterations = 200;
- while (0 < maxIterations--) {
- CloudState state = stateReader.getCloudState();
- Map<String,Slice> sliceMap = state.getSlices(collection);
- if (sliceMap != null && sliceMap.keySet().size() == i) {
- return;
- }
- Thread.sleep(100);
- }
- }
-
//wait until collections are available
private void waitForCollections(ZkStateReader stateReader, String... collections) throws InterruptedException, KeeperException {
int maxIterations = 100;
@@ -425,12 +411,10 @@ public class OverseerTest extends SolrTe
throw ke;
}
}
- //publish node state (recovering)
+
zkClient.setData(nodePath, ZkStateReader.toJSON(new CoreState[]{state}), true);
+ waitForCollections(reader, "collection1");
- //wait overseer assignment
- waitForSliceCount(reader, "collection1", 1);
-
assertEquals(reader.getCloudState().toString(), ZkStateReader.RECOVERING,
reader.getCloudState().getSlice("collection1", "shard1").getShards()
.get("node1_core1").get(ZkStateReader.STATE_PROP));
@@ -446,7 +430,6 @@ public class OverseerTest extends SolrTe
verifyStatus(reader, ZkStateReader.ACTIVE);
} finally {
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
if (zkClient != null) {
zkClient.close();
@@ -466,9 +449,12 @@ public class OverseerTest extends SolrTe
int maxIterations = 100;
String coreState = null;
while(maxIterations-->0) {
- coreState = reader.getCloudState().getSlice("collection1", "shard1").getShards().get("node1_core1").get(ZkStateReader.STATE_PROP);
- if(coreState.equals(expectedState)) {
- return;
+ Slice slice = reader.getCloudState().getSlice("collection1", "shard1");
+ if(slice!=null) {
+ coreState = slice.getShards().get("node1_core1").get(ZkStateReader.STATE_PROP);
+ if(coreState.equals(expectedState)) {
+ return;
+ }
}
Thread.sleep(50);
}
@@ -479,7 +465,6 @@ public class OverseerTest extends SolrTe
public void testOverseerFailure() throws Exception {
String zkDir = dataDir.getAbsolutePath() + File.separator
+ "zookeeper/server1/data";
-
ZkTestServer server = new ZkTestServer(zkDir);
SolrZkClient controllerClient = null;
@@ -501,11 +486,10 @@ public class OverseerTest extends SolrTe
overseerClient = electNewOverseer(server.getZkAddress());
+ Thread.sleep(1000);
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
-
- // wait overseer assignment
- waitForSliceCount(reader, "collection1", 1);
-
+
+ waitForCollections(reader, "collection1");
verifyStatus(reader, ZkStateReader.RECOVERING);
int version = getCloudStateVersion(controllerClient);
@@ -556,8 +540,6 @@ public class OverseerTest extends SolrTe
String zkDir = dataDir.getAbsolutePath() + File.separator
+ "zookeeper/server1/data";
- System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "2");
-
ZkTestServer server = new ZkTestServer(zkDir);
SolrZkClient controllerClient = null;
@@ -582,8 +564,7 @@ public class OverseerTest extends SolrTe
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
- // wait overseer assignment
- waitForSliceCount(reader, "collection1", 1);
+ waitForCollections(reader, "collection1");
verifyStatus(reader, ZkStateReader.RECOVERING);
@@ -598,10 +579,70 @@ public class OverseerTest extends SolrTe
reader.updateCloudState(true);
CloudState state = reader.getCloudState();
- assertEquals("more than 1 shard id was assigned to same core", 1, state.getSlices("collection1").size());
+
+ int numFound = 0;
+ for (Map<String,Slice> collection : state.getCollectionStates().values()) {
+ for (Slice slice : collection.values()) {
+ if (slice.getShards().get("node1_core1") != null) {
+ numFound++;
+ }
+ }
+ }
+ assertEquals("Shard was found in more than 1 times in CloudState", 1,
+ numFound);
+
+ } finally {
+ if (overseerClient != null) {
+ overseerClient.close();
+ }
+ if (mockController != null) {
+ mockController.close();
+ }
+
+ if (controllerClient != null) {
+ controllerClient.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ server.shutdown();
+ }
+ }
+ @Test
+ public void testPlaceholders() throws Exception {
+ String zkDir = dataDir.getAbsolutePath() + File.separator
+ + "zookeeper/server1/data";
+
+ ZkTestServer server = new ZkTestServer(zkDir);
+
+ SolrZkClient controllerClient = null;
+ SolrZkClient overseerClient = null;
+ ZkStateReader reader = null;
+ MockZKController mockController = null;
+
+ try {
+ server.run();
+ controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+
+ AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+ AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+ controllerClient.makePath(ZkStateReader.LIVE_NODES_ZKNODE, true);
+
+ reader = new ZkStateReader(controllerClient);
+ reader.createClusterStateWatchersAndUpdate();
+
+ mockController = new MockZKController(server.getZkAddress(), "node1");
+
+ overseerClient = electNewOverseer(server.getZkAddress());
+
+ mockController.publishState("core1", ZkStateReader.RECOVERING, 12);
+
+ waitForCollections(reader, "collection1");
+
+ assertEquals("Slicecount does not match", 12, reader.getCloudState().getSlices("collection1").size());
+
} finally {
- System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
if (overseerClient != null) {
overseerClient.close();
}