You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/04/11 14:52:18 UTC
svn commit: r1466860 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/admin/
solr/core/src/java/org/apache/solr/update/
solr/core/src/test/org/apache/solr/update/
Author: shalin
Date: Thu Apr 11 12:52:18 2013
New Revision: 1466860
URL: http://svn.apache.org/r1466860
Log:
SOLR-4685: Fix core admin SPLIT action to be useful with non-cloud setups
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1466860&r1=1466859&r2=1466860&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Thu Apr 11 12:52:18 2013
@@ -160,6 +160,8 @@ Bug Fixes
* SOLR-4699: The System admin handler should not assume a file system based data directory
location. (Mark Miller)
+* SOLR-4685: Fix core admin SPLIT action to be useful with non-cloud setups (shalin)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1466860&r1=1466859&r2=1466860&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Thu Apr 11 12:52:18 2013
@@ -231,18 +231,17 @@ public class CoreAdminHandler extends Re
*/
protected boolean handleSplitAction(SolrQueryRequest adminReq, SolrQueryResponse rsp) throws IOException {
SolrParams params = adminReq.getParams();
- // partitions=N (split into N partitions, leaving it up to solr what the ranges are and where to put them)
- // path - multiValued param, or comma separated param? Only creates indexes, not cores
-
List<DocRouter.Range> ranges = null;
- // boolean closeDirectories = true;
- // DirectoryFactory dirFactory = null;
String[] pathsArr = params.getParams("path");
String rangesStr = params.get("ranges"); // ranges=a-b,c-d,e-f
String[] newCoreNames = params.getParams("targetCore");
-
String cname = params.get(CoreAdminParams.CORE, "");
+
+ if ((pathsArr == null || pathsArr.length == 0) && (newCoreNames == null || newCoreNames.length == 0)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Either path or targetCore param must be specified");
+ }
+
log.info("Invoked split action for core: " + cname);
SolrCore core = coreContainer.getCore(cname);
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
@@ -251,20 +250,18 @@ public class CoreAdminHandler extends Re
try {
// TODO: allow use of rangesStr in the future
List<String> paths = null;
- int partitions = pathsArr != null ? pathsArr.length : params.getInt("partitions", 2);
-
- // TODO: if we don't know the real range of the current core, we should just
- // split on every other doc rather than hash.
- ClusterState clusterState = coreContainer.getZkController().getClusterState();
- String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
- DocCollection collection = clusterState.getCollection(collectionName);
- String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
- Slice slice = clusterState.getSlice(collectionName, sliceName);
- DocRouter.Range currentRange = slice.getRange() == null ?
- new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE) : slice.getRange();
+ int partitions = pathsArr != null ? pathsArr.length : newCoreNames.length;
- DocRouter hp = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
- ranges = hp.partitionRange(partitions, currentRange);
+ if (coreContainer.isZooKeeperAware()) {
+ ClusterState clusterState = coreContainer.getZkController().getClusterState();
+ String collectionName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName();
+ DocCollection collection = clusterState.getCollection(collectionName);
+ String sliceName = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
+ Slice slice = clusterState.getSlice(collectionName, sliceName);
+ DocRouter.Range currentRange = slice.getRange();
+ DocRouter hp = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
+ ranges = currentRange != null ? hp.partitionRange(partitions, currentRange) : null;
+ }
if (pathsArr == null) {
newCores = new ArrayList<SolrCore>(partitions);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1466860&r1=1466859&r2=1466860&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Thu Apr 11 12:52:18 2013
@@ -53,14 +53,21 @@ public class SolrIndexSplitter {
DocRouter.Range[] rangesArr; // same as ranges list, but an array for extra speed in inner loops
List<String> paths;
List<SolrCore> cores;
+ int numPieces;
+ int currPartition = 0;
public SolrIndexSplitter(SplitIndexCommand cmd) {
field = cmd.getReq().getSchema().getUniqueKeyField();
searcher = cmd.getReq().getSearcher();
ranges = cmd.ranges;
- rangesArr = ranges.toArray(new DocRouter.Range[ranges.size()]);
paths = cmd.paths;
cores = cmd.cores;
+ if (ranges == null) {
+ numPieces = paths != null ? paths.size() : cores.size();
+ } else {
+ numPieces = ranges.size();
+ rangesArr = ranges.toArray(new DocRouter.Range[ranges.size()]);
+ }
}
public void split() throws IOException {
@@ -68,7 +75,7 @@ public class SolrIndexSplitter {
List<AtomicReaderContext> leaves = searcher.getTopReaderContext().leaves();
List<OpenBitSet[]> segmentDocSets = new ArrayList<OpenBitSet[]>(leaves.size());
- log.info("SolrIndexSplitter: partitions=" + ranges.size() + " segments="+leaves.size());
+ log.info("SolrIndexSplitter: partitions=" + numPieces + " segments="+leaves.size());
for (AtomicReaderContext readerContext : leaves) {
assert readerContext.ordInParent == segmentDocSets.size(); // make sure we're going in order
@@ -83,8 +90,8 @@ public class SolrIndexSplitter {
// - would be more efficient on the read side, but prob less efficient merging
IndexReader[] subReaders = new IndexReader[leaves.size()];
- for (int partitionNumber=0; partitionNumber<ranges.size(); partitionNumber++) {
- log.info("SolrIndexSplitter: partition #" + partitionNumber + " range=" + ranges.get(partitionNumber));
+ for (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {
+ log.info("SolrIndexSplitter: partition #" + partitionNumber + (ranges != null ? " range=" + ranges.get(partitionNumber) : ""));
for (int segmentNumber = 0; segmentNumber<subReaders.length; segmentNumber++) {
subReaders[segmentNumber] = new LiveDocsReader( leaves.get(segmentNumber), segmentDocSets.get(segmentNumber)[partitionNumber] );
@@ -101,7 +108,7 @@ public class SolrIndexSplitter {
} else {
SolrCore core = searcher.getCore();
String path = paths.get(partitionNumber);
- iw = SolrIndexWriter.create("SplittingIndexWriter"+partitionNumber + " " + ranges.get(partitionNumber), path,
+ iw = SolrIndexWriter.create("SplittingIndexWriter"+partitionNumber + (ranges != null ? " " + ranges.get(partitionNumber) : ""), path,
core.getDirectoryFactory(), true, core.getSchema(),
core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
}
@@ -130,7 +137,7 @@ public class SolrIndexSplitter {
OpenBitSet[] split(AtomicReaderContext readerContext) throws IOException {
AtomicReader reader = readerContext.reader();
- OpenBitSet[] docSets = new OpenBitSet[ranges.size()];
+ OpenBitSet[] docSets = new OpenBitSet[numPieces];
for (int i=0; i<docSets.length; i++) {
docSets[i] = new OpenBitSet(reader.maxDoc());
}
@@ -158,9 +165,14 @@ public class SolrIndexSplitter {
for (;;) {
int doc = docsEnum.nextDoc();
if (doc == DocsEnum.NO_MORE_DOCS) break;
- for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
- if (rangesArr[i].includes(hash)) {
- docSets[i].fastSet(doc);
+ if (ranges == null) {
+ docSets[currPartition].fastSet(doc);
+ currPartition = (currPartition + 1) % numPieces;
+ } else {
+ for (int i=0; i<rangesArr.length; i++) { // inner-loop: use array here for extra speed.
+ if (rangesArr[i].includes(hash)) {
+ docSets[i].fastSet(doc);
+ }
}
}
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java?rev=1466860&r1=1466859&r2=1466860&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java Thu Apr 11 12:52:18 2013
@@ -41,7 +41,7 @@ import java.io.UnsupportedEncodingExcept
import java.util.List;
public class SolrIndexSplitterTest extends SolrTestCaseJ4 {
- File indexDir1 = null, indexDir2 = null;
+ File indexDir1 = null, indexDir2 = null, indexDir3 = null;
@BeforeClass
public static void beforeClass() throws Exception {
@@ -58,6 +58,8 @@ public class SolrIndexSplitterTest exten
+ "_testSplit1");
indexDir2 = new File(TEMP_DIR, this.getClass().getName()
+ "_testSplit2");
+ indexDir3 = new File(TEMP_DIR, this.getClass().getName()
+ + "_testSplit3");
if (indexDir1.exists()) {
FileUtils.deleteDirectory(indexDir1);
@@ -68,6 +70,11 @@ public class SolrIndexSplitterTest exten
FileUtils.deleteDirectory(indexDir2);
}
assertTrue("Failed to mkdirs indexDir2 for split index", indexDir2.mkdirs());
+
+ if (indexDir3.exists()) {
+ FileUtils.deleteDirectory(indexDir3);
+ }
+ assertTrue("Failed to mkdirs indexDir3 for split index", indexDir3.mkdirs());
}
@Test
@@ -162,6 +169,53 @@ public class SolrIndexSplitterTest exten
}
}
+ @Test
+ public void testSplitAlternately() throws Exception {
+ LocalSolrQueryRequest request = null;
+ Directory directory = null;
+ try {
+ // add an even number of docs
+ int max = (1 + random().nextInt(10)) * 3;
+ log.info("Adding {} number of documents", max);
+ for (int i = 0; i < max; i++) {
+ assertU(adoc("id", String.valueOf(i)));
+ }
+ assertU(commit());
+
+ request = lrf.makeRequest("q", "dummy");
+
+ SplitIndexCommand command = new SplitIndexCommand(request,
+ Lists.newArrayList(indexDir1.getAbsolutePath(), indexDir2.getAbsolutePath(), indexDir3.getAbsolutePath()), null, null);
+ new SolrIndexSplitter(command).split();
+
+ directory = h.getCore().getDirectoryFactory().get(indexDir1.getAbsolutePath(),
+ DirectoryFactory.DirContext.DEFAULT, h.getCore().getSolrConfig().indexConfig.lockType);
+ DirectoryReader reader = DirectoryReader.open(directory);
+ assertEquals("split index1 has wrong number of documents", max / 3, reader.numDocs());
+ reader.close();
+ h.getCore().getDirectoryFactory().release(directory);
+ directory = h.getCore().getDirectoryFactory().get(indexDir2.getAbsolutePath(),
+ DirectoryFactory.DirContext.DEFAULT, h.getCore().getSolrConfig().indexConfig.lockType);
+ reader = DirectoryReader.open(directory);
+ assertEquals("split index2 has wrong number of documents", max / 3, reader.numDocs());
+ reader.close();
+ h.getCore().getDirectoryFactory().release(directory);
+ directory = h.getCore().getDirectoryFactory().get(indexDir3.getAbsolutePath(),
+ DirectoryFactory.DirContext.DEFAULT, h.getCore().getSolrConfig().indexConfig.lockType);
+ reader = DirectoryReader.open(directory);
+ assertEquals("split index3 has wrong number of documents", max / 3, reader.numDocs());
+ reader.close();
+ h.getCore().getDirectoryFactory().release(directory);
+ directory = null;
+ } finally {
+ if (request != null) request.close(); // decrefs the searcher
+ if (directory != null) {
+ // perhaps an assert failed, release the directory
+ h.getCore().getDirectoryFactory().release(directory);
+ }
+ }
+ }
+
private List<DocRouter.Range> getRanges(String id1, String id2) throws UnsupportedEncodingException {
// find minHash/maxHash hash ranges
byte[] bytes = id1.getBytes("UTF-8");