You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/07/12 10:16:54 UTC
svn commit: r1145483 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ lucene/contrib/
lucene/contrib/misc/src/java/org/apache/lucene/index/
lucene/contrib/misc/src/test/or...
Author: simonw
Date: Tue Jul 12 08:16:54 2011
New Revision: 1145483
URL: http://svn.apache.org/viewvc?rev=1145483&view=rev
Log:
LUCENE-3296: Enable passing a IndexWriterConfig into PKIndexSplitter
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1145483&r1=1145482&r2=1145483&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Tue Jul 12 08:16:54 2011
@@ -28,7 +28,10 @@ New Features
Removed contrib/wordnet. (Robert Muir, Mike McCandless)
API Changes
-
+ * LUCENE-3296: PKIndexSplitter & MultiPassIndexSplitter now have version
+ constructors. PKIndexSplitter accepts a IndexWriterConfig for each of
+ the target indexes. (Simon Willnauer, Jason Rutherglen)
+
Bug Fixes
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=1145483&r1=1145482&r2=1145483&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Tue Jul 12 08:16:54 2011
@@ -46,7 +46,6 @@ import org.apache.lucene.util.Version;
* break up such document groups.
*/
public class MultiPassIndexSplitter {
-
/**
* Split source index into multiple parts.
* @param input source index, can be read-only, can have deletions, can have
@@ -56,8 +55,25 @@ public class MultiPassIndexSplitter {
* increasing ranges of document id-s. If false, source document id-s will be
* assigned in a deterministic round-robin fashion to one of the output splits.
* @throws IOException
+ * @deprecated use {@link #split(Version, IndexReader, Directory[], boolean)} instead.
+ * This method will be removed in Lucene 4.0.
*/
+ @Deprecated
public void split(IndexReader input, Directory[] outputs, boolean seq) throws IOException {
+ split(Version.LUCENE_CURRENT, input, outputs, seq);
+ }
+
+ /**
+ * Split source index into multiple parts.
+ * @param input source index, can be read-only, can have deletions, can have
+ * multiple segments (or multiple readers).
+ * @param outputs list of directories where the output parts will be stored.
+ * @param seq if true, then the source index will be split into equal
+ * increasing ranges of document id-s. If false, source document id-s will be
+ * assigned in a deterministic round-robin fashion to one of the output splits.
+ * @throws IOException
+ */
+ public void split(Version version, IndexReader input, Directory[] outputs, boolean seq) throws IOException {
if (outputs == null || outputs.length < 2) {
throw new IOException("Invalid number of outputs.");
}
@@ -96,7 +112,7 @@ public class MultiPassIndexSplitter {
}
}
IndexWriter w = new IndexWriter(outputs[i], new IndexWriterConfig(
- Version.LUCENE_CURRENT,
+ version,
new WhitespaceAnalyzer(Version.LUCENE_CURRENT))
.setOpenMode(OpenMode.CREATE));
System.err.println("Writing part " + (i + 1) + " ...");
@@ -106,6 +122,7 @@ public class MultiPassIndexSplitter {
System.err.println("Done.");
}
+ @SuppressWarnings("deprecation")
public static void main(String[] args) throws Exception {
if (args.length < 5) {
System.err.println("Usage: MultiPassIndexSplitter -out <outputDir> -num <numParts> [-seq] <inputIndex1> [<inputIndex2 ...]");
@@ -169,7 +186,7 @@ public class MultiPassIndexSplitter {
} else {
input = new MultiReader(indexes.toArray(new IndexReader[indexes.size()]));
}
- splitter.split(input, dirs, seq);
+ splitter.split(Version.LUCENE_CURRENT, input, dirs, seq);
}
/**
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java?rev=1145483&r1=1145482&r2=1145483&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java Tue Jul 12 08:16:54 2011
@@ -37,17 +37,54 @@ public class PKIndexSplitter {
private final Filter docsInFirstIndex;
private final Directory input;
private final Directory dir1;
- private final Directory dir2;
+ private final Directory dir2;
+ private final IndexWriterConfig config1;
+ private final IndexWriterConfig config2;
/**
* Split an index based on a {@link Filter}. All documents that match the filter
* are sent to dir1, remaining ones to dir2.
+ * @deprecated use {@link #PKIndexSplitter(Version, Directory, Directory, Directory, Filter)} instead.
+ * This constructor will be removed in Lucene 4.0.
*/
+ @Deprecated
public PKIndexSplitter(Directory input, Directory dir1, Directory dir2, Filter docsInFirstIndex) {
+ this(Version.LUCENE_CURRENT, input, dir1, dir2, docsInFirstIndex);
+ }
+
+ /**
+ * Split an index based on a given primary key term
+ * and a 'middle' term. If the middle term is present, it's
+ * sent to dir2.
+ * @deprecated use {@link #PKIndexSplitter(Version, Directory, Directory, Directory, Term)}
+ * instead. This constructor will be removed in Lucene 4.0.
+ */
+ @Deprecated
+ public PKIndexSplitter(Directory input, Directory dir1, Directory dir2, Term midTerm) {
+ this(input, dir1, dir2,
+ new TermRangeFilter(midTerm.field(), null, midTerm.text(), true, false));
+ }
+
+ /**
+ * Split an index based on a {@link Filter}. All documents that match the filter
+ * are sent to dir1, remaining ones to dir2.
+ */
+ public PKIndexSplitter(Version version, Directory input, Directory dir1, Directory dir2, Filter docsInFirstIndex) {
+ this(input, dir1, dir2, docsInFirstIndex, newDefaultConfig(version), newDefaultConfig(version));
+ }
+
+ private static IndexWriterConfig newDefaultConfig(Version version) {
+ return new IndexWriterConfig(version, null).setOpenMode(OpenMode.CREATE);
+ }
+
+ public PKIndexSplitter(Directory input, Directory dir1,
+ Directory dir2, Filter docsInFirstIndex, IndexWriterConfig config1, IndexWriterConfig config2) {
this.input = input;
this.dir1 = dir1;
this.dir2 = dir2;
this.docsInFirstIndex = docsInFirstIndex;
+ this.config1 = config1;
+ this.config2 = config2;
}
/**
@@ -55,27 +92,33 @@ public class PKIndexSplitter {
* and a 'middle' term. If the middle term is present, it's
* sent to dir2.
*/
- public PKIndexSplitter(Directory input, Directory dir1, Directory dir2, Term midTerm) {
- this(input, dir1, dir2,
+ public PKIndexSplitter(Version version, Directory input, Directory dir1, Directory dir2, Term midTerm) {
+ this(version, input, dir1, dir2,
new TermRangeFilter(midTerm.field(), null, midTerm.text(), true, false));
}
+ public PKIndexSplitter(Directory input, Directory dir1,
+ Directory dir2, Term midTerm, IndexWriterConfig config1, IndexWriterConfig config2) {
+ this(input, dir1, dir2,
+ new TermRangeFilter(midTerm.field(), null, midTerm.text(), true, false), config1, config2);
+ }
+
public void split() throws IOException {
boolean success = false;
IndexReader reader = IndexReader.open(input);
try {
- createIndex(dir1, reader, docsInFirstIndex, false);
- createIndex(dir2, reader, docsInFirstIndex, true);
+ // pass an individual config in here since one config can not be reused!
+ createIndex(config1, dir1, reader, docsInFirstIndex, false);
+ createIndex(config2, dir2, reader, docsInFirstIndex, true);
success = true;
} finally {
IOUtils.closeSafely(!success, reader);
}
}
- private void createIndex(Directory target, IndexReader reader, Filter preserveFilter, boolean negateFilter) throws IOException {
+ private void createIndex(IndexWriterConfig config, Directory target, IndexReader reader, Filter preserveFilter, boolean negateFilter) throws IOException {
boolean success = false;
- IndexWriter w = new IndexWriter(target, new IndexWriterConfig(
- Version.LUCENE_CURRENT, null).setOpenMode(OpenMode.CREATE));
+ IndexWriter w = new IndexWriter(target, config);
try {
w.addIndexes(new DocumentFilteredIndexReader(reader, preserveFilter, negateFilter));
success = true;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java?rev=1145483&r1=1145482&r2=1145483&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java Tue Jul 12 08:16:54 2011
@@ -66,7 +66,7 @@ public class TestMultiPassIndexSplitter
newDirectory(),
newDirectory()
};
- splitter.split(input, dirs, false);
+ splitter.split(TEST_VERSION_CURRENT, input, dirs, false);
IndexReader ir;
ir = IndexReader.open(dirs[0], true);
assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1); // rounding error
@@ -111,7 +111,7 @@ public class TestMultiPassIndexSplitter
newDirectory(),
newDirectory()
};
- splitter.split(input, dirs, true);
+ splitter.split(TEST_VERSION_CURRENT, input, dirs, true);
IndexReader ir;
ir = IndexReader.open(dirs[0], true);
assertTrue(ir.numDocs() - NUM_DOCS / 3 <= 1);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java?rev=1145483&r1=1145482&r2=1145483&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java Tue Jul 12 08:16:54 2011
@@ -67,7 +67,9 @@ public class TestPKIndexSplitter extends
private void checkSplitting(Directory dir, Term splitTerm, int leftCount, int rightCount) throws Exception {
Directory dir1 = newDirectory();
Directory dir2 = newDirectory();
- PKIndexSplitter splitter = new PKIndexSplitter(dir, dir1, dir2, splitTerm);
+ PKIndexSplitter splitter = new PKIndexSplitter(dir, dir1, dir2, splitTerm,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)),
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
splitter.split();
IndexReader ir1 = IndexReader.open(dir1);