You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/10/05 15:00:37 UTC
svn commit: r1529444 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/handler/admin/
core/src/test/org/apache/solr/cloud/
solrj/src/java/org/apache/solr/common/params/
Author: shalin
Date: Sat Oct 5 13:00:36 2013
New Revision: 1529444
URL: http://svn.apache.org/r1529444
Log:
SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges within the shard's hash range
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sat Oct 5 13:00:36 2013
@@ -84,7 +84,10 @@ New Features
(shalin)
* SOLR-5274: Allow JettySolrRunner SSL config to be specified via a constructor.
- (Mark Miller)
+ (Mark Miller)
+
+* SOLR-5300: Shards can be split by specifying arbitrary number of hash ranges
+ within the shard's hash range. (shalin)
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Sat Oct 5 13:00:36 2013
@@ -484,9 +484,32 @@ public class OverseerCollectionProcessor
range = new PlainIdRouter().fullRange();
}
- // todo: fixed to two partitions?
- // todo: accept the range as a param to api?
- List<DocRouter.Range> subRanges = router.partitionRange(2, range);
+ List<DocRouter.Range> subRanges = null;
+ String rangesStr = message.getStr(CoreAdminParams.RANGES);
+ if (rangesStr != null) {
+ String[] ranges = rangesStr.split(",");
+ if (ranges.length == 0 || ranges.length == 1) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split a shard");
+ } else {
+ subRanges = new ArrayList<DocRouter.Range>(ranges.length);
+ for (int i = 0; i < ranges.length; i++) {
+ String r = ranges[i];
+ try {
+ subRanges.add(DocRouter.DEFAULT.fromString(r));
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Exception in parsing hexadecimal hash range: " + r, e);
+ }
+ if (!subRanges.get(i).isSubsetOf(range)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Specified hash range: " + r + " is not a subset of parent shard's range: " + range.toString());
+ }
+ }
+ }
+ } else {
+ // todo: fixed to two partitions?
+ subRanges = router.partitionRange(2, range);
+ }
+
try {
List<String> subSlices = new ArrayList<String>(subRanges.size());
List<String> subShardNames = new ArrayList<String>(subRanges.size());
@@ -579,6 +602,7 @@ public class OverseerCollectionProcessor
String subShardName = subShardNames.get(i);
params.add(CoreAdminParams.TARGET_CORE, subShardName);
}
+ params.set(CoreAdminParams.RANGES, rangesStr);
sendShardRequest(parentShardLeader.getNodeName(), params);
collectShardResponses(results, true, "SPLITSHARD failed to invoke SPLIT core admin command");
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Sat Oct 5 13:00:36 2013
@@ -370,12 +370,15 @@ public class CollectionsHandler extends
String name = req.getParams().required().get("collection");
// TODO : add support for multiple shards
String shard = req.getParams().required().get("shard");
- // TODO : add support for shard range
+ String rangesStr = req.getParams().get(CoreAdminParams.RANGES);
Map<String,Object> props = new HashMap<String,Object>();
props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.SPLITSHARD);
props.put("collection", name);
props.put(ZkStateReader.SHARD_ID_PROP, shard);
+ if (rangesStr != null) {
+ props.put(CoreAdminParams.RANGES, rangesStr);
+ }
ZkNodeProps m = new ZkNodeProps(props);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Sat Oct 5 13:00:36 2013
@@ -231,7 +231,22 @@ public class CoreAdminHandler extends Re
List<DocRouter.Range> ranges = null;
String[] pathsArr = params.getParams("path");
- String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
+ String rangesStr = params.get(CoreAdminParams.RANGES); // ranges=a-b,c-d,e-f
+ if (rangesStr != null) {
+ String[] rangesArr = rangesStr.split(",");
+ if (rangesArr.length == 0 || rangesArr.length == 1) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "There must be at least two ranges specified to split an index");
+ } else {
+ ranges = new ArrayList<DocRouter.Range>(rangesArr.length);
+ for (String r : rangesArr) {
+ try {
+ ranges.add(DocRouter.DEFAULT.fromString(r));
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Exception parsing hexadecimal hash range: " + r, e);
+ }
+ }
+ }
+ }
String[] newCoreNames = params.getParams("targetCore");
String cname = params.get(CoreAdminParams.CORE, "");
@@ -257,9 +272,11 @@ public class CoreAdminHandler extends Re
DocCollection collection = clusterState.getCollection(collectionName);
String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
Slice slice = clusterState.getSlice(collectionName, sliceName);
- DocRouter.Range currentRange = slice.getRange();
router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
- ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
+ if (ranges == null) {
+ DocRouter.Range currentRange = slice.getRange();
+ ranges = currentRange != null ? router.partitionRange(partitions, currentRange) : null;
+ }
Map m = (Map) collection.get(DOC_ROUTER);
if (m != null) {
routeFieldName = (String) m.get("field");
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java Sat Oct 5 13:00:36 2013
@@ -135,7 +135,7 @@ public class ChaosMonkeyShardSplitTest e
killerThread.start();
killCounter.incrementAndGet();
- splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
+ splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null);
log.info("Layout after split: \n");
printLayout();
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java Sat Oct 5 13:00:36 2013
@@ -18,7 +18,6 @@ package org.apache.solr.cloud;
*/
import org.apache.http.params.CoreConnectionPNames;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
@@ -45,6 +44,7 @@ import org.junit.Before;
import java.io.IOException;
import java.net.MalformedURLException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -123,7 +123,16 @@ public class ShardSplitTest extends Basi
final DocRouter router = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter();
Slice shard1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
- final List<DocRouter.Range> ranges = router.partitionRange(2, shard1Range);
+ List<DocRouter.Range> subRanges = new ArrayList<DocRouter.Range>();
+ if (usually()) {
+ List<DocRouter.Range> ranges = router.partitionRange(4, shard1Range);
+ // 75% of range goes to shard1_0 and the rest to shard1_1
+ subRanges.add(new DocRouter.Range(ranges.get(0).min, ranges.get(2).max));
+ subRanges.add(ranges.get(3));
+ } else {
+ subRanges = router.partitionRange(2, shard1Range);
+ }
+ final List<DocRouter.Range> ranges = subRanges;
final int[] docCounts = new int[ranges.size()];
int numReplicas = shard1.getReplicas().size();
@@ -167,7 +176,7 @@ public class ShardSplitTest extends Basi
try {
for (int i = 0; i < 3; i++) {
try {
- splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
+ splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges);
log.info("Layout after split: \n");
printLayout();
break;
@@ -252,7 +261,7 @@ public class ShardSplitTest extends Basi
for (int i = 0; i < 3; i++) {
try {
- splitShard(collectionName, SHARD1);
+ splitShard(collectionName, SHARD1, null);
break;
} catch (HttpSolrServer.RemoteSolrException e) {
if (e.code() != 500) {
@@ -339,11 +348,21 @@ public class ShardSplitTest extends Basi
}
}
- protected void splitShard(String collection, String shardId) throws SolrServerException, IOException {
+ protected void splitShard(String collection, String shardId, List<DocRouter.Range> subRanges) throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.SPLITSHARD.toString());
params.set("collection", collection);
params.set("shard", shardId);
+ if (subRanges != null) {
+ StringBuilder ranges = new StringBuilder();
+ for (int i = 0; i < subRanges.size(); i++) {
+ DocRouter.Range subRange = subRanges.get(i);
+ ranges.append(subRange.toString());
+ if (i < subRanges.size() - 1)
+ ranges.append(",");
+ }
+ params.set("ranges", ranges.toString());
+ }
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java?rev=1529444&r1=1529443&r2=1529444&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java Sat Oct 5 13:00:36 2013
@@ -83,6 +83,9 @@ public abstract class CoreAdminParams
/** The target core to which a split index should be written to
* Multiple targetCores can be specified by multiple targetCore parameters */
public final static String TARGET_CORE = "targetCore";
+
+ /** The hash ranges to be used to split a shard or an index */
+ public final static String RANGES = "ranges";
public static final String ROLES = "roles";