You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/08/31 00:34:21 UTC
svn commit: r1379196 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/core/ lucene/demo/ lucene/facet/ lucene/grouping/
luce...
Author: yonik
Date: Thu Aug 30 22:34:19 2012
New Revision: 1379196
URL: http://svn.apache.org/viewvc?rev=1379196&view=rev
Log:
SOLR-3755: basic shard splitting code
Added:
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
- copied unchanged from r1379195, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
- copied unchanged from r1379195, lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
lucene/dev/branches/branch_4x/solr/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/lib/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Thu Aug 30 22:34:19 2012
@@ -19,10 +19,13 @@ package org.apache.solr.handler.admin;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -36,6 +39,7 @@ import org.apache.solr.cloud.ZkControlle
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.HashPartitioner;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -57,6 +61,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.MergeIndexesCommand;
+import org.apache.solr.update.SplitIndexCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.util.NumberUtils;
@@ -171,6 +176,11 @@ public class CoreAdminHandler extends Re
break;
}
+ case SPLIT: {
+ doPersist = this.handleSplitAction(req, rsp);
+ break;
+ }
+
case PREPRECOVERY: {
this.handleWaitForStateAction(req, rsp);
break;
@@ -202,6 +212,62 @@ public class CoreAdminHandler extends Re
rsp.setHttpCaching(false);
}
+
+ protected boolean handleSplitAction(SolrQueryRequest adminReq, SolrQueryResponse rsp) throws IOException {
+ SolrParams params = adminReq.getParams();
+ // partitions=N (split into N partitions, leaving it up to solr what the ranges are and where to put them)
+ // path - multiValued param, or comma separated param? Only creates indexes, not cores
+
+ List<HashPartitioner.Range> ranges = null;
+ // boolean closeDirectories = true;
+ // DirectoryFactory dirFactory = null;
+
+
+ String cname = params.get(CoreAdminParams.CORE, "");
+ SolrCore core = coreContainer.getCore(cname);
+ SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
+ try {
+
+ String[] pathsArr = params.getParams("path");
+ List<String> paths = null;
+
+ String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
+
+
+ // dirFactory = core.getDirectoryFactory();
+
+
+ if (pathsArr != null) {
+
+ paths = Arrays.asList(pathsArr);
+
+ if (rangesStr == null) {
+ HashPartitioner hp = new HashPartitioner();
+ // should this be static?
+ // TODO: use real range if we know it. If we don't know it, we should prob
+ // split on every other doc rather than on a hash?
+ ranges = hp.partitionRange(pathsArr.length, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ }
+
+ }
+
+
+ SplitIndexCommand cmd = new SplitIndexCommand(req, paths, ranges);
+ core.getUpdateHandler().split(cmd);
+
+ } catch (Exception e) {
+ log.error("ERROR executing split:", e);
+ throw new RuntimeException(e);
+
+ } finally {
+ if (req != null) req.close();
+ if (core != null) core.close();
+ }
+
+ return false;
+ }
+
+
protected boolean handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
SolrParams params = req.getParams();
String cname = params.required().get(CoreAdminParams.CORE);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Thu Aug 30 22:34:19 2012
@@ -738,6 +738,13 @@ public class DirectUpdateHandler2 extend
}
}
+ @Override
+ public void split(SplitIndexCommand cmd) throws IOException {
+ // TODO: do a commit first?
+ SolrIndexSplitter splitter = new SolrIndexSplitter(cmd);
+ splitter.split();
+ }
+
/////////////////////////////////////////////////////////////////////
// SolrInfoMBean stuff: Statistics and Module Info
/////////////////////////////////////////////////////////////////////
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/UpdateHandler.java Thu Aug 30 22:34:19 2012
@@ -179,4 +179,6 @@ public abstract class UpdateHandler impl
{
optimizeCallbacks.add( listener );
}
+
+ public abstract void split(SplitIndexCommand cmd) throws IOException;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java Thu Aug 30 22:34:19 2012
@@ -27,12 +27,25 @@ public class TestHashPartitioner extends
public void testMapHashes() throws Exception {
HashPartitioner hp = new HashPartitioner();
-
- for (int i = 1; i <= 30000; i++) {
- List<Range> ranges = hp.partitionRange(i);
-
+ List<Range> ranges;
+
+ // make sure the partitioner uses the "natural" boundaries and doesn't suffer from an off-by-one
+ ranges = hp.partitionRange(2, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ assertEquals(Integer.MIN_VALUE, ranges.get(0).min);
+ assertEquals(0x80000000, ranges.get(0).min);
+ assertEquals(0xffffffff, ranges.get(0).max);
+ assertEquals(0x00000000, ranges.get(1).min);
+ assertEquals(0x7fffffff, ranges.get(1).max);
+
+ ranges = hp.partitionRange(2, 0, 0x7fffffff);
+ assertEquals(0x00000000, ranges.get(0).min);
+ assertEquals(0x3fffffff, ranges.get(0).max);
+ assertEquals(0x40000000, ranges.get(1).min);
+ assertEquals(0x7fffffff, ranges.get(1).max);
+
+ for (int i = 1; i <= 30000; i += 13) {
+ ranges = hp.partitionRange(i, Integer.MIN_VALUE, Integer.MAX_VALUE);
assertEquals(i, ranges.size());
-
assertTrue("First range does not start before " + Integer.MIN_VALUE
+ " it is:" + ranges.get(0).min,
ranges.get(0).min <= Integer.MIN_VALUE);
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java Thu Aug 30 22:34:19 2012
@@ -226,7 +226,7 @@ public class ClusterState implements JSO
shardList.addAll(shards);
Collections.sort(shardList);
- ranges = hp.partitionRange(shards.size());
+ ranges = hp.partitionRange(shards.size(), Integer.MIN_VALUE, Integer.MAX_VALUE);
rangeInfo.ranges = ranges;
rangeInfo.shardList = shardList;
@@ -243,7 +243,7 @@ public class ClusterState implements JSO
int cnt = 0;
for (Range range : rangInfo.ranges) {
- if (hash < range.max) {
+ if (range.includes(hash)) {
return rangInfo.shardList.get(cnt);
}
cnt++;
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/HashPartitioner.java Thu Aug 30 22:34:19 2012
@@ -25,39 +25,57 @@ import java.util.List;
*
*/
public class HashPartitioner {
-
+
+ // Hash ranges can't currently "wrap" - i.e. max must be greater or equal to min.
public static class Range {
- public long min;
- public long max;
+ public int min; // inclusive
+ public int max; // inclusive
- public Range(long min, long max) {
+ public Range(int min, int max) {
this.min = min;
this.max = max;
}
+
+ public boolean includes(int hash) {
+ return hash >= min && hash <= max;
+ }
+
+ public String toString() {
+ return Integer.toHexString(min) + '-' + Integer.toHexString(max);
+ }
+
+ public static Range fromString(String range) {
+ return null; // TODO
+ }
}
+
/**
- * works up to 65537 before requested num of ranges is one short
- *
+ *
* @param partitions
* @return Range for each partition
*/
- public List<Range> partitionRange(int partitions) {
- // some hokey code to partition the int space
- long range = Integer.MAX_VALUE + (Math.abs((long) Integer.MIN_VALUE));
- long srange = range / partitions;
-
+ public List<Range> partitionRange(int partitions, int min, int max) {
+ assert max >= min;
+ long range = (long)max - (long)min;
+ long srange = Math.max(1, range / partitions);
+
List<Range> ranges = new ArrayList<Range>(partitions);
-
- long end = 0;
- long start = Integer.MIN_VALUE;
-
- while (end < Integer.MAX_VALUE) {
+
+ long start = min;
+ long end = start;
+
+ while (end < max) {
end = start + srange;
- ranges.add(new Range(start, end));
+ // make last range always end exactly on MAX_VALUE
+ if (ranges.size() == partitions - 1) {
+ end = max;
+ }
+ ranges.add(new Range((int)start, (int)end));
start = end + 1L;
}
-
+
return ranges;
}
+
}
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java?rev=1379196&r1=1379195&r2=1379196&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java Thu Aug 30 22:34:19 2012
@@ -92,7 +92,8 @@ public interface CoreAdminParams
SWAP,
RENAME,
MERGEINDEXES,
- PREPRECOVERY,
+ SPLIT,
+ PREPRECOVERY,
REQUESTRECOVERY,
REQUESTSYNCSHARD;