You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/02/09 14:59:12 UTC
svn commit: r1242324 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/handler/admin/
core/src/test/org/apache/solr/cloud/
solrj/src/java/org/apache/solr/client/solrj/request/
solrj/src/java/org/apache/s...
Author: markrmiller
Date: Thu Feb 9 13:59:11 2012
New Revision: 1242324
URL: http://svn.apache.org/viewvc?rev=1242324&view=rev
Log:
SOLR-3108: Error in SolrCloud's replica lookup code when replica's are hosted in same Solr instance (test)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Feb 9 13:59:11 2012
@@ -108,6 +108,8 @@ New Features
(Soheb Mahmood, Alex Cowell, Mark Miller)
SOLR-2622: ShowFileRequestHandler does not work in SolrCloud mode.
(Stefan Matheis, Mark Miller)
+ SOLR-3108: Error in SolrCloud's replica lookup code when replica's are hosted in same Solr instance.
+ (Bruno Dumon, Sami Siren, Mark Miller)
* SOLR-1566: Transforming documents in the ResponseWriters. This will allow
for more complex results in responses and open the door for function queries
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/AssignShard.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/AssignShard.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/AssignShard.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/AssignShard.java Thu Feb 9 13:59:11 2012
@@ -26,8 +26,6 @@ import java.util.Map;
import org.apache.solr.common.cloud.CloudState;
import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.zookeeper.KeeperException;
public class AssignShard {
@@ -38,10 +36,10 @@ public class AssignShard {
* @param state
* @return the assigned shard id
*/
- public static String assignShard(String collection, CloudState state) {
-
- int shards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP,1);
-
+ public static String assignShard(String collection, CloudState state, Integer numShards) {
+ if (numShards == null) {
+ numShards = 1;
+ }
String returnShardId = null;
Map<String, Slice> sliceMap = state.getSlices(collection);
@@ -51,7 +49,7 @@ public class AssignShard {
List<String> shardIdNames = new ArrayList<String>(sliceMap.keySet());
- if (shardIdNames.size() < shards) {
+ if (shardIdNames.size() < numShards) {
return "shard" + (shardIdNames.size() + 1);
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java Thu Feb 9 13:59:11 2012
@@ -24,6 +24,7 @@ public class CloudDescriptor {
private String collectionName;
private SolrParams params;
private String roles = "";
+ private Integer numShards;
public void setShardId(String shardId) {
this.shardId = shardId;
@@ -57,4 +58,14 @@ public class CloudDescriptor {
public void setParams(SolrParams params) {
this.params = params;
}
+
+ // setting only matters on core creation
+ public Integer getNumShards() {
+ return numShards;
+ }
+
+ public void setNumShards(int numShards) {
+ this.numShards = numShards;
+ }
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java Thu Feb 9 13:59:11 2012
@@ -198,7 +198,7 @@ public class Overseer implements NodeSta
}
if(shardId==null) {
//request new shardId
- shardId = AssignShard.assignShard(collection, state);
+ shardId = AssignShard.assignShard(collection, state, coreState.getNumShards());
}
Map<String,String> props = new HashMap<String,String>();
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu Feb 9 13:59:11 2012
@@ -844,9 +844,12 @@ public final class ZkController {
private void doPublish(String shardZkNodeName, String coreName,
Map<String,String> props, CloudDescriptor cloudDesc) {
-
+ Integer numShards = cloudDesc.getNumShards();
+ if (numShards == null) {
+ numShards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP);
+ }
CoreState coreState = new CoreState(coreName,
- cloudDesc.getCollectionName(), props);
+ cloudDesc.getCollectionName(), props, numShards);
coreStates.put(shardZkNodeName, coreState);
final String nodePath = "/node_states/" + getNodeName();
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Thu Feb 9 13:59:11 2012
@@ -26,7 +26,6 @@ import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.SolrServerException;
@@ -354,6 +353,10 @@ public class CoreAdminHandler extends Re
opts = params.get(CoreAdminParams.SHARD);
if (opts != null)
cd.setShardId(opts);
+
+ Integer numShards = params.getInt(ZkStateReader.NUM_SHARDS_PROP);
+ if (numShards != null)
+ cd.setNumShards(numShards);
}
// Process all property.name=value parameters and set them as name=value core properties
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Thu Feb 9 13:59:11 2012
@@ -61,6 +61,7 @@ public class BasicDistributedZkTest exte
String invalidField="ignore_exception__invalid_field_not_in_schema";
private Map<String,List<SolrServer>> otherCollectionClients = new HashMap<String,List<SolrServer>>();
+ private Map<String,List<SolrServer>> oneInstanceCollectionClients = new HashMap<String,List<SolrServer>>();
public BasicDistributedZkTest() {
fixShardCount = true;
@@ -243,12 +244,72 @@ public class BasicDistributedZkTest exte
// query("q","matchesnothing","fl","*,score", "debugQuery", "true");
testMultipleCollections();
+ testANewCollectionInOneInstance();
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
+ private void testANewCollectionInOneInstance() throws Exception {
+ String collection = "oneInstanceCollection";
+ List<SolrServer> collectionClients = new ArrayList<SolrServer>();
+ SolrServer client = clients.get(0);
+ oneInstanceCollectionClients.put(collection, collectionClients);
+ String baseUrl = ((CommonsHttpSolrServer) client).getBaseURL();
+ createCollection(collection, collectionClients, baseUrl, 1);
+ createCollection(collection, collectionClients, baseUrl, 2);
+ createCollection(collection, collectionClients, baseUrl, 3);
+ createCollection(collection, collectionClients, baseUrl, 4);
+
+ SolrServer client1 = createNewSolrServer(collection + "1", baseUrl);
+ SolrServer client2 = createNewSolrServer(collection + "2", baseUrl);
+ SolrServer client3 = createNewSolrServer(collection + "3", baseUrl);
+ SolrServer client4 = createNewSolrServer(collection + "4", baseUrl);
+
+ client2.add(getDoc(id, "1"));
+ client3.add(getDoc(id, "2"));
+ client4.add(getDoc(id, "3"));
+
+ client1.commit();
+ SolrQuery query = new SolrQuery("*:*");
+ query.set("distrib", false);
+ long oneDocs = client1.query(query).getResults().getNumFound();
+ long twoDocs = client2.query(query).getResults().getNumFound();
+ long threeDocs = client3.query(query).getResults().getNumFound();
+ long fourDocs = client4.query(query).getResults().getNumFound();
+
+ query.set("collection", collection);
+ query.set("distrib", true);
+ long allDocs = solrj.query(query).getResults().getNumFound();
+
+// System.out.println("1:" + oneDocs);
+// System.out.println("2:" + twoDocs);
+// System.out.println("3:" + threeDocs);
+// System.out.println("4:" + fourDocs);
+// System.out.println("All Docs:" + allDocs);
+
+ assertEquals(oneDocs, threeDocs);
+ assertEquals(twoDocs, fourDocs);
+ assertNotSame(oneDocs, twoDocs);
+ assertEquals(3, allDocs);
+ }
+
+ private void createCollection(String collection,
+ List<SolrServer> collectionClients, String baseUrl, int num)
+ throws MalformedURLException, SolrServerException, IOException {
+ CommonsHttpSolrServer server = new CommonsHttpSolrServer(
+ baseUrl);
+ Create createCmd = new Create();
+ createCmd.setCoreName(collection + num);
+ createCmd.setCollection(collection);
+ createCmd.setNumShards(2);
+ createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ + collection + num);
+ server.request(createCmd);
+ collectionClients.add(createNewSolrServer(collection, baseUrl));
+ }
+
private void testMultipleCollections() throws MalformedURLException,
SolrServerException, IOException, Exception {
// create another 2 collections and search across them
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Thu Feb 9 13:59:11 2012
@@ -86,12 +86,12 @@ public class OverseerTest extends SolrTe
}
}
- public void publishState(String coreName, String stateName) throws KeeperException, InterruptedException{
+ public void publishState(String coreName, String stateName, int numShards) throws KeeperException, InterruptedException{
HashMap<String,String> coreProps = new HashMap<String,String>();
coreProps.put(ZkStateReader.STATE_PROP, stateName);
coreProps.put(ZkStateReader.NODE_NAME_PROP, nodeName);
coreProps.put(ZkStateReader.CORE_NAME_PROP, coreName);
- CoreState state = new CoreState(coreName, "collection1", coreProps);
+ CoreState state = new CoreState(coreName, "collection1", coreProps, numShards);
final String statePath = Overseer.STATES_NODE + "/" + nodeName;
zkClient.setData(statePath, ZkStateReader.toJSON(new CoreState[] {state}), true);
}
@@ -397,8 +397,6 @@ public class OverseerTest extends SolrTe
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
zkClient.makePath("/live_nodes", true);
- System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "2");
-
//live node
String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1";
zkClient.makePath(nodePath,CreateMode.EPHEMERAL, true);
@@ -416,7 +414,7 @@ public class OverseerTest extends SolrTe
coreProps.put(ZkStateReader.CORE_NAME_PROP, "core1");
coreProps.put(ZkStateReader.ROLES_PROP, "");
coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING);
- CoreState state = new CoreState("core1", "collection1", coreProps);
+ CoreState state = new CoreState("core1", "collection1", coreProps, 2);
nodePath = "/node_states/node1";
@@ -441,7 +439,7 @@ public class OverseerTest extends SolrTe
coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.ACTIVE);
coreProps.put(ZkStateReader.SHARD_ID_PROP, "shard1");
- state = new CoreState("core1", "collection1", coreProps);
+ state = new CoreState("core1", "collection1", coreProps, 2);
zkClient.setData(nodePath, ZkStateReader.toJSON(new CoreState[]{state}), true);
@@ -503,7 +501,7 @@ public class OverseerTest extends SolrTe
overseerClient = electNewOverseer(server.getZkAddress());
- mockController.publishState("core1", ZkStateReader.RECOVERING);
+ mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
// wait overseer assignment
waitForSliceCount(reader, "collection1", 1);
@@ -512,7 +510,7 @@ public class OverseerTest extends SolrTe
int version = getCloudStateVersion(controllerClient);
- mockController.publishState("core1", ZkStateReader.ACTIVE);
+ mockController.publishState("core1", ZkStateReader.ACTIVE, 1);
while(version == getCloudStateVersion(controllerClient));
@@ -521,7 +519,7 @@ public class OverseerTest extends SolrTe
overseerClient.close();
Thread.sleep(1000); //wait for overseer to get killed
- mockController.publishState("core1", ZkStateReader.RECOVERING);
+ mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
version = getCloudStateVersion(controllerClient);
overseerClient = electNewOverseer(server.getZkAddress());
@@ -582,7 +580,7 @@ public class OverseerTest extends SolrTe
overseerClient = electNewOverseer(server.getZkAddress());
- mockController.publishState("core1", ZkStateReader.RECOVERING);
+ mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
// wait overseer assignment
waitForSliceCount(reader, "collection1", 1);
@@ -594,7 +592,7 @@ public class OverseerTest extends SolrTe
int version = getCloudStateVersion(controllerClient);
mockController = new MockZKController(server.getZkAddress(), "node1");
- mockController.publishState("core1", ZkStateReader.RECOVERING);
+ mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
while (version == getCloudStateVersion(controllerClient));
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java Thu Feb 9 13:59:11 2012
@@ -26,6 +26,7 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
@@ -49,6 +50,8 @@ public class CoreAdminRequest extends So
protected String configName = null;
protected String schemaName = null;
protected String dataDir = null;
+ protected String collection;
+ private Integer numShards;
public Create() {
action = CoreAdminAction.CREATE;
@@ -58,12 +61,15 @@ public class CoreAdminRequest extends So
public void setSchemaName(String schema) { this.schemaName = schema; }
public void setConfigName(String config) { this.configName = config; }
public void setDataDir(String dataDir) { this.dataDir = dataDir; }
+ public void setCollection(String collection) { this.collection = collection; }
+ public void setNumShards(int numShards) {this.numShards = numShards;}
public String getInstanceDir() { return instanceDir; }
public String getSchemaName() { return schemaName; }
public String getConfigName() { return configName; }
public String getDataDir() { return dataDir; }
-
+ public String getCollection() { return collection; }
+
@Override
public SolrParams getParams() {
if( action == null ) {
@@ -86,8 +92,15 @@ public class CoreAdminRequest extends So
if (dataDir != null) {
params.set( CoreAdminParams.DATA_DIR, dataDir);
}
+ if (collection != null) {
+ params.set( CoreAdminParams.COLLECTION, collection);
+ }
+ if (numShards != null) {
+ params.set( ZkStateReader.NUM_SHARDS_PROP, numShards);
+ }
return params;
}
+
}
public static class PrepRecovery extends CoreAdminRequest {
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java?rev=1242324&r1=1242323&r2=1242324&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java Thu Feb 9 13:59:11 2012
@@ -32,11 +32,14 @@ public class CoreState implements JSONWr
this.properties = Collections.unmodifiableMap(props);
}
- public CoreState(String coreName, String collectionName, Map<String,String> properties) {
+ public CoreState(String coreName, String collectionName, Map<String,String> properties, Integer numShards) {
HashMap<String,String> props = new HashMap<String,String>();
props.putAll(properties);
props.put(ZkStateReader.COLLECTION_PROP, collectionName);
props.put(ZkStateReader.CORE_NAME_PROP, coreName);
+ if (numShards != null) {
+ props.put("num_shards", Integer.toString(numShards));
+ }
this.properties = Collections.unmodifiableMap(props);
}
@@ -93,4 +96,12 @@ public class CoreState implements JSONWr
return "coll:" + getCollectionName() + " core:" + getCoreName() + " props:" + properties;
}
+ public Integer getNumShards() {
+ String numShards = properties.get("num_shards");
+ if (numShards == null) {
+ return null;
+ }
+ return Integer.parseInt(numShards);
+ }
+
}