You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/12/10 23:34:56 UTC
svn commit: r1212906 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/ lucene/contrib/misc/src/java/org/apache/lucene/index/
lucene/contrib/misc/src/test/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/index/
Author: uschindler
Date: Sat Dec 10 22:34:56 2011
New Revision: 1212906
URL: http://svn.apache.org/viewvc?rev=1212906&view=rev
Log:
LUCENE-3626: Make PKIndexSplitter and MultiPassIndexSplitter work per segment
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1212906&r1=1212905&r2=1212906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Sat Dec 10 22:34:56 2011
@@ -5,6 +5,12 @@ http://s.apache.org/luceneversions
======================= Lucene 3.6.0 ================
+Changes in backwards compatibility policy
+
+ * LUCENE-3626: The internal implementation classes in PKIndexSplitter
+ and MultiPassIndexSplitter were made private as they now work
+ per segment. (Uwe Schindler)
+
New Features
* LUCENE-3596: DirectoryTaxonomyWriter extensions can override createIndexWriterConfig()
@@ -20,6 +26,11 @@ API Changes
not take position overlaps into account while recalculating norms.
(Uwe Schindler, Robert Muir)
+Changes in runtime behavior:
+
+ * LUCENE-3626: PKIndexSplitter and MultiPassIndexSplitter now work
+ per segment. (Uwe Schindler)
+
Bug Fixes
* LUCENE-3600: BlockJoinQuery now supports parent docs that have no
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java?rev=1212906&r1=1212905&r2=1212906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Sat Dec 10 22:34:56 2011
@@ -22,11 +22,11 @@ import java.io.IOException;
import java.util.ArrayList;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;
/**
@@ -194,12 +194,32 @@ public class MultiPassIndexSplitter {
* Instead, deletions are buffered in a bitset and overlaid with the original
* list of deletions.
*/
- public static class FakeDeleteIndexReader extends FilterIndexReader {
+ private static final class FakeDeleteIndexReader extends MultiReader {
+
+ public FakeDeleteIndexReader(IndexReader reader) throws IOException {
+ super(initSubReaders(reader), false /* dont close */);
+ }
+
+ private static IndexReader[] initSubReaders(IndexReader reader) throws IOException {
+ final ArrayList<IndexReader> subs = new ArrayList<IndexReader>();
+ new ReaderUtil.Gather(reader) {
+ @Override
+ protected void add(int base, IndexReader r) {
+ subs.add(new FakeDeleteAtomicIndexReader(r));
+ }
+ }.run();
+ return subs.toArray(new IndexReader[subs.size()]);
+ }
+
+ }
+
+ private static final class FakeDeleteAtomicIndexReader extends FilterIndexReader {
FixedBitSet dels;
FixedBitSet oldDels;
- public FakeDeleteIndexReader(IndexReader in) {
+ public FakeDeleteAtomicIndexReader(IndexReader in) {
super(in);
+ assert in.getSequentialSubReaders() == null;
dels = new FixedBitSet(in.maxDoc());
if (in.hasDeletions()) {
oldDels = new FixedBitSet(in.maxDoc());
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java?rev=1212906&r1=1212905&r2=1212906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java Sat Dec 10 22:34:56 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.ArrayList;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
@@ -27,6 +28,7 @@ import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;
/**
@@ -122,9 +124,15 @@ public class PKIndexSplitter {
private void createIndex(IndexWriterConfig config, Directory target, IndexReader reader, Filter preserveFilter, boolean negateFilter) throws IOException {
boolean success = false;
- IndexWriter w = new IndexWriter(target, config);
+ final IndexWriter w = new IndexWriter(target, config);
try {
- w.addIndexes(new DocumentFilteredIndexReader(reader, preserveFilter, negateFilter));
+ final ArrayList<IndexReader> leaves = new ArrayList<IndexReader>();
+ ReaderUtil.gatherSubReaders(leaves, reader);
+ final IndexReader[] subReaders = new IndexReader[leaves.size()];
+ for (int i = 0; i < subReaders.length; i++) {
+ subReaders[i] = new DocumentFilteredAtomicIndexReader(leaves.get(i), preserveFilter, negateFilter);
+ }
+ w.addIndexes(subReaders);
success = true;
} finally {
if (success) {
@@ -135,13 +143,13 @@ public class PKIndexSplitter {
}
}
- public static class DocumentFilteredIndexReader extends FilterIndexReader {
+ private static class DocumentFilteredAtomicIndexReader extends FilterIndexReader {
final FixedBitSet readerDels;
final int numDocs;
- public DocumentFilteredIndexReader(IndexReader reader, Filter preserveFilter, boolean negateFilter) throws IOException {
+ public DocumentFilteredAtomicIndexReader(IndexReader reader, Filter preserveFilter, boolean negateFilter) throws IOException {
super(reader);
-
+ assert in.getSequentialSubReaders() == null;
final FixedBitSet bits = new FixedBitSet(in.maxDoc());
final DocIdSet docs = preserveFilter.getDocIdSet(in);
if (docs != null) {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java?rev=1212906&r1=1212905&r2=1212906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java Sat Dec 10 22:34:56 2011
@@ -32,13 +32,14 @@ public class TestMultiPassIndexSplitter
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
Document doc;
for (int i = 0; i < NUM_DOCS; i++) {
doc = new Document();
doc.add(newField("id", i + "", Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.add(newField("f", i + " " + i, Field.Store.YES, Field.Index.ANALYZED));
w.addDocument(doc);
+ if (i%3==0) w.commit();
}
w.close();
input = IndexReader.open(dir, false);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java?rev=1212906&r1=1212905&r2=1212906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestPKIndexSplitter.java Sat Dec 10 22:34:56 2011
@@ -36,14 +36,16 @@ public class TestPKIndexSplitter extends
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))
- .setOpenMode(OpenMode.CREATE));
+ .setOpenMode(OpenMode.CREATE).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
for (int x = 0; x < 11; x++) {
Document doc = createDocument(x, "1", 3, format);
w.addDocument(doc);
+ if (x%3==0) w.commit();
}
for (int x = 11; x < 20; x++) {
Document doc = createDocument(x, "2", 3, format);
w.addDocument(doc);
+ if (x%3==0) w.commit();
}
w.close();
@@ -54,7 +56,7 @@ public class TestPKIndexSplitter extends
// delete some documents
w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))
- .setOpenMode(OpenMode.APPEND));
+ .setOpenMode(OpenMode.APPEND).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
w.deleteDocuments(midTerm);
w.deleteDocuments(new Term("id", format.format(2)));
w.close();
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1212906&r1=1212905&r2=1212906&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java Sat Dec 10 22:34:56 2011
@@ -35,10 +35,10 @@ import org.apache.lucene.util.MapBackedS
/** An IndexReader which reads multiple indexes, appending
* their content. */
public class MultiReader extends IndexReader implements Cloneable {
- protected IndexReader[] subReaders;
- private int[] starts; // 1st docno for each segment
+ protected final IndexReader[] subReaders;
+ protected final int[] starts; // 1st docno for each segment
private boolean[] decrefOnClose; // remember which subreaders to decRef on close
- private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
+ private final Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
private int maxDoc = 0;
private int numDocs = -1;
private boolean hasDeletions = false;
@@ -51,7 +51,7 @@ public class MultiReader extends IndexRe
* @param subReaders set of (sub)readers
*/
public MultiReader(IndexReader... subReaders) {
- initialize(subReaders, true);
+ this(subReaders, true);
}
/**
@@ -63,10 +63,6 @@ public class MultiReader extends IndexRe
* @param subReaders set of (sub)readers
*/
public MultiReader(IndexReader[] subReaders, boolean closeSubReaders) {
- initialize(subReaders, closeSubReaders);
- }
-
- private void initialize(IndexReader[] subReaders, boolean closeSubReaders) {
this.subReaders = subReaders.clone();
starts = new int[subReaders.length + 1]; // build starts array
decrefOnClose = new boolean[subReaders.length];
@@ -282,7 +278,7 @@ public class MultiReader extends IndexRe
numDocs = -1; // invalidate cache
}
- private int readerIndex(int n) { // find reader for doc n:
+ protected int readerIndex(int n) { // find reader for doc n:
return DirectoryReader.readerIndex(n, this.starts, this.subReaders.length);
}