You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/12/08 04:25:26 UTC
svn commit: r1043289 - in /lucene/dev/branches/docvalues/lucene: ./ contrib/
contrib/highlighter/src/test/
contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
contrib/misc/src/java/org/apache/lucene/index/
src/java/org/apache/lucene/ana...
Author: simonw
Date: Wed Dec 8 03:25:25 2010
New Revision: 1043289
URL: http://svn.apache.org/viewvc?rev=1043289&view=rev
Log:
Added DocValuesIndexing#addIndexes() and merged LUCENE-2802 & LUCENE-2805
Modified:
lucene/dev/branches/docvalues/lucene/ (props changed)
lucene/dev/branches/docvalues/lucene/CHANGES.txt
lucene/dev/branches/docvalues/lucene/build.xml (props changed)
lucene/dev/branches/docvalues/lucene/contrib/ (props changed)
lucene/dev/branches/docvalues/lucene/contrib/CHANGES.txt (props changed)
lucene/dev/branches/docvalues/lucene/contrib/highlighter/src/test/ (props changed)
lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (props changed)
lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java (props changed)
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (props changed)
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/document/TestDateTools.java (props changed)
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (props changed)
Modified: lucene/dev/branches/docvalues/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/CHANGES.txt?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/lucene/CHANGES.txt Wed Dec 8 03:25:25 2010
@@ -127,6 +127,15 @@ Changes in Runtime Behavior
changed to enable unmapping by default if supported by the JRE.
(Mike McCandless, Uwe Schindler, Robert Muir)
+* LUCENE-2790: LogMergePolicy.useCompoundFile's logic now factors in noCFSRatio
+ to determine whether the passed in segment should be compound.
+ (Shai Erera, Earwin Burrfoot)
+
+* LUCENE-2805: IndexWriter now increments the index version on every change to
+ the index instead of for every commit. Committing or closing the IndexWriter
+ without any changes to the index will not cause any index version increment.
+ (Simon Willnauer, Mike Mccandless)
+
API Changes
* LUCENE-2302, LUCENE-1458, LUCENE-2111, LUCENE-2514: Terms are no longer
@@ -335,6 +344,17 @@ Bug fixes
* LUCENE-2633: PackedInts Packed32 and Packed64 did not support internal
structures larger than 256MB (Toke Eskildsen via Mike McCandless)
+* LUCENE-2803: The FieldCache can miss values if an entry for a reader
+ with more document deletions is requested before a reader with fewer
+ deletions, provided they share some segments. (yonik)
+
+* LUCENE-2802: NRT DirectoryReader returned incorrect values from
+ getVersion, isOptimized, getCommitUserData, getIndexCommit and isCurrent due
+ to a mutable reference to the IndexWriters SegmentInfos.
+ (Simon Willnauer, Earwin Burrfoot)
+
+
+
======================= Lucene 3.x (not yet released) =======================
Changes in backwards compatibility policy
Modified: lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java Wed Dec 8 03:25:25 2010
@@ -134,6 +134,7 @@ public class IndexSplitter {
int idx = getIdx(n);
infos.remove(idx);
}
+ infos.changed();
infos.commit(fsDir);
}
@@ -152,6 +153,7 @@ public class IndexSplitter {
copyFile(srcFile, destFile);
}
}
+ destInfos.changed();
destInfos.commit(destFSDir);
// System.out.println("destDir:"+destDir.getAbsolutePath());
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/CheckIndex.java Wed Dec 8 03:25:25 2010
@@ -879,6 +879,7 @@ public class CheckIndex {
public void fixIndex(Status result) throws IOException {
if (result.partial)
throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)");
+ result.newSegments.changed();
result.newSegments.commit(result.dir);
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Wed Dec 8 03:25:25 2010
@@ -55,8 +55,7 @@ class DirectoryReader extends IndexReade
private IndexDeletionPolicy deletionPolicy;
private Lock writeLock;
- private SegmentInfos segmentInfos;
- private SegmentInfos segmentInfosStart;
+ private final SegmentInfos segmentInfos;
private boolean stale;
private final int termInfosIndexDivisor;
@@ -106,7 +105,6 @@ class DirectoryReader extends IndexReade
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
-
if (codecs == null) {
this.codecs = CodecProvider.getDefault();
} else {
@@ -145,8 +143,7 @@ class DirectoryReader extends IndexReade
DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
- segmentInfos = infos;
- segmentInfosStart = (SegmentInfos) infos.clone();
+ segmentInfos = (SegmentInfos) infos.clone();// make sure we clone otherwise we share mutable state with IW
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (codecs == null) {
this.codecs = CodecProvider.getDefault();
@@ -770,6 +767,7 @@ class DirectoryReader extends IndexReade
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
segmentInfos, null, null, codecs);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
+ segmentInfos.changed();
// Checkpoint the state we are about to change, in
// case we have to roll back:
@@ -782,7 +780,6 @@ class DirectoryReader extends IndexReade
// Sync all files we just wrote
directory.sync(segmentInfos.files(directory, false));
-
segmentInfos.commit(directory);
success = true;
} finally {
@@ -860,7 +857,7 @@ class DirectoryReader extends IndexReade
// we loaded SegmentInfos from the directory
return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion();
} else {
- return writer.nrtIsCurrent(segmentInfosStart);
+ return writer.nrtIsCurrent(segmentInfos);
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Dec 8 03:25:25 2010
@@ -741,6 +741,7 @@ public class IndexWriter implements Clos
// Record that we have a change (zero out all
// segments) pending:
changeCount++;
+ segmentInfos.changed();
} else {
segmentInfos.read(directory, codecs);
@@ -757,6 +758,7 @@ public class IndexWriter implements Clos
oldInfos.read(directory, commit.getSegmentsFileName(), codecs);
segmentInfos.replace(oldInfos);
changeCount++;
+ segmentInfos.changed();
if (infoStream != null)
message("init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
}
@@ -774,12 +776,14 @@ public class IndexWriter implements Clos
conf.getIndexDeletionPolicy(),
segmentInfos, infoStream, docWriter, codecs);
- if (deleter.startingCommitDeleted)
+ if (deleter.startingCommitDeleted) {
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
changeCount++;
+ segmentInfos.changed();
+ }
docWriter.setMaxBufferedDeleteTerms(conf.getMaxBufferedDeleteTerms());
docWriter.setRAMBufferSizeMB(conf.getRAMBufferSizeMB());
@@ -1537,6 +1541,7 @@ public class IndexWriter implements Clos
// name that was previously returned which can cause
// problems at least with ConcurrentMergeScheduler.
changeCount++;
+ segmentInfos.changed();
return "_" + Integer.toString(segmentInfos.counter++, Character.MAX_RADIX);
}
}
@@ -2038,6 +2043,7 @@ public class IndexWriter implements Clos
// Mark that the index has changed
++changeCount;
+ segmentInfos.changed();
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteAll");
} finally {
@@ -2119,6 +2125,7 @@ public class IndexWriter implements Clos
*/
private synchronized void checkpoint() throws IOException {
changeCount++;
+ segmentInfos.changed();
deleter.checkpoint(segmentInfos, false);
}
@@ -3697,6 +3704,7 @@ public class IndexWriter implements Clos
}
toSync.remove(toSync.size()-1);
changeCount++;
+ segmentInfos.changed();
}
}
assert filesExist(toSync);
@@ -3829,17 +3837,7 @@ public class IndexWriter implements Clos
}
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
- if (!infos.equals(segmentInfos)) {
- // if any structural changes (new segments), we are
- // stale
- return false;
- } else if (infos.getGeneration() != segmentInfos.getGeneration()) {
- // if any commit took place since we were opened, we
- // are stale
- return false;
- } else {
- return !docWriter.anyChanges();
- }
+ return infos.version == segmentInfos.version && !docWriter.anyChanges();
}
synchronized boolean isClosed() {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Wed Dec 8 03:25:25 2010
@@ -696,7 +696,6 @@ public final class SegmentInfos extends
void updateGeneration(SegmentInfos other) {
lastGeneration = other.lastGeneration;
generation = other.generation;
- version = other.version;
}
final void rollbackCommit(Directory dir) throws IOException {
@@ -727,7 +726,12 @@ public final class SegmentInfos extends
* segments file, but writes an invalid checksum at the
* end, so that it is not visible to readers. Once this
* is called you must call {@link #finishCommit} to complete
- * the commit or {@link #rollbackCommit} to abort it. */
+ * the commit or {@link #rollbackCommit} to abort it.
+ * <p>
+ * Note: {@link #changed()} should be called prior to this
+ * method if changes have been made to this {@link SegmentInfos} instance
+ * </p>
+ **/
final void prepareCommit(Directory dir) throws IOException {
if (pendingSegnOutput != null)
throw new IllegalStateException("prepareCommit was already called");
@@ -811,7 +815,12 @@ public final class SegmentInfos extends
}
/** Writes & syncs to the Directory dir, taking care to
- * remove the segments file on exception */
+ * remove the segments file on exception
+ * <p>
+ * Note: {@link #changed()} should be called prior to this
+ * method if changes have been made to this {@link SegmentInfos} instance
+ * </p>
+ **/
final void commit(Directory dir) throws IOException {
prepareCommit(dir);
finishCommit(dir);
@@ -862,4 +871,10 @@ public final class SegmentInfos extends
}
return count;
}
+
+ /** Call this before committing if changes have been made to the
+ * segments. */
+ public void changed() {
+ version++;
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Wed Dec 8 03:25:25 2010
@@ -51,8 +51,7 @@ public class DefaultSegmentInfosWriter e
throws IOException {
IndexOutput out = createOutput(dir, segmentFileName);
out.writeInt(FORMAT_CURRENT); // write FORMAT
- out.writeLong(++infos.version); // every write changes
- // the index
+ out.writeLong(infos.version);
out.writeInt(infos.counter); // write counter
out.writeInt(infos.size()); // write infos
for (SegmentInfo si : infos) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Wed Dec 8 03:25:25 2010
@@ -244,7 +244,7 @@ class FixedDerefBytesImpl {
@Override
public int nextDoc() throws IOException {
- if (pos < valueCount) {
+ if (pos >= valueCount) {
return pos = NO_MORE_DOCS;
}
return advance(pos + 1);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java Wed Dec 8 03:25:25 2010
@@ -142,8 +142,9 @@ public class MultiDocValues extends DocV
+ " must be > than the current doc " + currentDoc;
int relativeDoc = target - currentStart;
do {
- if (target >= maxDoc) // we are beyond max doc
+ if (target >= maxDoc) {// we are beyond max doc
return currentDoc = NO_MORE_DOCS;
+ }
if (target >= currentMax) {
final int idx = ReaderUtil.subIndex(target, starts);
currentEnum.close();
@@ -152,9 +153,9 @@ public class MultiDocValues extends DocV
currentStart = docValuesIdx[idx].start;
currentMax = currentStart + docValuesIdx[idx].length;
relativeDoc = target - currentStart;
- } else {
- return currentDoc = currentStart + currentEnum.advance(relativeDoc);
}
+ target = currentMax; // make sure that we advance to the next enum if the current is exhausted
+
} while ((relativeDoc = currentEnum.advance(relativeDoc)) == NO_MORE_DOCS);
return currentDoc = currentStart + relativeDoc;
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Wed Dec 8 03:25:25 2010
@@ -61,7 +61,54 @@ public class TestIndexWriterReader exten
}
return count;
}
-
+
+ public void testAddCloseOpen() throws IOException {
+ Directory dir1 = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
+
+ IndexWriter writer = new IndexWriter(dir1, iwc);
+ for (int i = 0; i < 97 ; i++) {
+ IndexReader reader = writer.getReader();
+ if (i == 0) {
+ writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ } else {
+ int previous = random.nextInt(i);
+ // a check if the reader is current here could fail since there might be
+ // merges going on.
+ switch (random.nextInt(5)) {
+ case 0:
+ case 1:
+ case 2:
+ writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ break;
+ case 3:
+ writer.updateDocument(new Term("id", "" + previous), createDocument(
+ previous, "x", 1 + random.nextInt(5)));
+ break;
+ case 4:
+ writer.deleteDocuments(new Term("id", "" + previous));
+ }
+ }
+ assertFalse(reader.isCurrent());
+ reader.close();
+ }
+ writer.optimize(); // make sure all merging is done etc.
+ IndexReader reader = writer.getReader();
+ writer.commit(); // no changes that are not visible to the reader
+ assertTrue(reader.isCurrent());
+ writer.close();
+ assertTrue(reader.isCurrent()); // all changes are visible to the reader
+ iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
+ writer = new IndexWriter(dir1, iwc);
+ assertTrue(reader.isCurrent());
+ writer.addDocument(createDocument(1, "x", 1+random.nextInt(5)));
+ assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one
+ writer.close();
+ assertFalse(reader.isCurrent()); // segments written
+ reader.close();
+ dir1.close();
+ }
+
public void testUpdateDocument() throws Exception {
boolean optimize = true;
@@ -128,6 +175,44 @@ public class TestIndexWriterReader exten
dir1.close();
}
+ public void testIsCurrent() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
+
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ writer.close();
+
+ iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
+ writer = new IndexWriter(dir, iwc);
+ doc = new Document();
+ doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED));
+ IndexReader nrtReader = writer.getReader();
+ assertTrue(nrtReader.isCurrent());
+ writer.addDocument(doc);
+ assertFalse(nrtReader.isCurrent()); // should see the changes
+ writer.optimize(); // make sure we don't have a merge going on
+ assertFalse(nrtReader.isCurrent());
+ nrtReader.close();
+
+ IndexReader dirReader = IndexReader.open(dir);
+ nrtReader = writer.getReader();
+
+ assertTrue(dirReader.isCurrent());
+ assertTrue(nrtReader.isCurrent()); // nothing was committed yet so we are still current
+ assertEquals(2, nrtReader.maxDoc()); // sees the actual document added
+ assertEquals(1, dirReader.maxDoc());
+ writer.close(); // close is actually a commit both should see the changes
+ assertTrue(nrtReader.isCurrent());
+ assertFalse(dirReader.isCurrent()); // this reader has been opened before the writer was closed / committed
+
+ dirReader.close();
+ nrtReader.close();
+ dir.close();
+ }
+
/**
* Test using IW.addIndexes
*
@@ -171,7 +256,7 @@ public class TestIndexWriterReader exten
assertTrue(r1.isCurrent());
writer.commit();
- assertFalse(r1.isCurrent());
+ assertTrue(r1.isCurrent()); // we have seen all changes - no change after opening the NRT reader
assertEquals(200, r1.maxDoc());
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java?rev=1043289&r1=1043288&r2=1043289&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java Wed Dec 8 03:25:25 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.index.values;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
@@ -67,21 +68,14 @@ import org.junit.Before;
*/
public class TestDocValuesIndexing extends LuceneTestCase {
/*
- * TODO:
- * Roadmap to land on trunk
- *
- * - Add documentation for:
- * - Source and ValuesEnum
- * - DocValues
- * - ValuesField
- * - ValuesAttribute
- * - Values
- * - Add @lucene.experimental to all necessary classes
- * - add test for unoptimized case with deletes
- * - add a test for addIndexes
- * - split up existing testcases and give them meaningfull names
- * - run RAT
- * - add tests for FieldComparator FloatIndexValuesComparator vs. FloatValuesComparator etc.
+ * TODO: Roadmap to land on trunk
+ *
+ * - Add documentation for: - Source and ValuesEnum - DocValues - ValuesField
+ * - ValuesAttribute - Values - Add @lucene.experimental to all necessary
+ * classes - add test for unoptimized case with deletes - add a test for
+ * addIndexes - split up existing testcases and give them meaningfull names -
+ * run RAT - add tests for FieldComparator FloatIndexValuesComparator vs.
+ * FloatValuesComparator etc.
*/
private DocValuesCodec docValuesCodec;
@@ -90,18 +84,20 @@ public class TestDocValuesIndexing exten
@Before
public void setUp() throws Exception {
super.setUp();
- String defaultFieldCodec = CodecProvider.getDefault().getDefaultFieldCodec();
+ String defaultFieldCodec = CodecProvider.getDefault()
+ .getDefaultFieldCodec();
provider = new CodecProvider();
- docValuesCodec = new DocValuesCodec(CodecProvider.getDefault().lookup(defaultFieldCodec));
+ docValuesCodec = new DocValuesCodec(CodecProvider.getDefault().lookup(
+ defaultFieldCodec));
provider.register(docValuesCodec);
provider.setDefaultFieldCodec(docValuesCodec.name);
}
-
-
+
/*
* Simple test case to show how to use the API
*/
- public void testDocValuesSimple() throws CorruptIndexException, IOException, ParseException {
+ public void testDocValuesSimple() throws CorruptIndexException, IOException,
+ ParseException {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, writerConfig(false));
for (int i = 0; i < 5; i++) {
@@ -114,14 +110,15 @@ public class TestDocValuesIndexing exten
}
writer.commit();
writer.optimize(true);
-
+
writer.close();
-
+
IndexReader reader = IndexReader.open(dir, null, true, 1, provider);
assertTrue(reader.isOptimized());
-
+
IndexSearcher searcher = new IndexSearcher(reader);
- QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "docId", new MockAnalyzer());
+ QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "docId",
+ new MockAnalyzer());
TopDocs search = searcher.search(parser.parse("0 OR 1 OR 2 OR 3 OR 4"), 10);
assertEquals(5, search.totalHits);
ScoreDoc[] scoreDocs = search.scoreDocs;
@@ -175,6 +172,79 @@ public class TestDocValuesIndexing exten
runTestNumerics(writerConfig(false), true);
}
+ public void testAddIndexes() throws IOException {
+ int valuesPerIndex = 10;
+ List<Type> values = Arrays.asList(Type.values());
+ Collections.shuffle(values, random);
+ Type first = values.get(0);
+ Type second = values.get(1);
+ String msg = "[first=" + first.name() + ", second=" + second.name() + "]";
+ // index first index
+ Directory d_1 = newDirectory();
+ IndexWriter w_1 = new IndexWriter(d_1, writerConfig(random.nextBoolean()));
+ indexValues(w_1, valuesPerIndex, first, values, false, 7);
+ w_1.commit();
+ assertEquals(valuesPerIndex, w_1.maxDoc());
+ _TestUtil.checkIndex(d_1, w_1.getConfig().getCodecProvider());
+
+ // index second index
+ Directory d_2 = newDirectory();
+ IndexWriter w_2 = new IndexWriter(d_2, writerConfig(random.nextBoolean()));
+ indexValues(w_2, valuesPerIndex, second, values, false, 7);
+ w_2.commit();
+ assertEquals(valuesPerIndex, w_2.maxDoc());
+ _TestUtil.checkIndex(d_2, w_2.getConfig().getCodecProvider());
+
+ Directory target = newDirectory();
+ IndexWriter w = new IndexWriter(target, writerConfig(random.nextBoolean()));
+ IndexReader r_1 = IndexReader.open(w_1);
+ IndexReader r_2 = IndexReader.open(w_2);
+ if (random.nextBoolean()) {
+ w.addIndexes(d_1, d_2);
+ } else {
+ w.addIndexes(r_1, r_2);
+ }
+ w.optimize();
+ w.commit();
+
+ _TestUtil.checkIndex(target, w.getConfig().getCodecProvider());
+ assertEquals(valuesPerIndex * 2, w.maxDoc());
+
+ // check values
+
+ IndexReader merged = IndexReader.open(w);
+ DocValuesEnum vE_1 = getValuesEnum(getDocValues(r_1, first.name()));
+ DocValuesEnum vE_2 = getValuesEnum(getDocValues(r_2, second.name()));
+ DocValuesEnum vE_1_merged = getValuesEnum(getDocValues(merged, first.name()));
+ DocValuesEnum vE_2_merged = getValuesEnum(getDocValues(merged, second
+ .name()));
+ if (second == Type.BYTES_VAR_STRAIGHT || second == Type.BYTES_FIXED_STRAIGHT) {
+ assertEquals(msg, valuesPerIndex-1, vE_2_merged.advance(valuesPerIndex-1));
+ }
+ for (int i = 0; i < valuesPerIndex; i++) {
+ assertEquals(msg, i, vE_1.nextDoc());
+ assertEquals(msg, i, vE_1_merged.nextDoc());
+
+ assertEquals(msg, i, vE_2.nextDoc());
+ assertEquals(msg, i + valuesPerIndex, vE_2_merged.nextDoc());
+ }
+ assertEquals(msg, DocValuesEnum.NO_MORE_DOCS, vE_1.nextDoc());
+ assertEquals(msg, DocValuesEnum.NO_MORE_DOCS, vE_2.nextDoc());
+ assertEquals(msg, DocValuesEnum.NO_MORE_DOCS, vE_1_merged.advance(valuesPerIndex*2));
+ assertEquals(msg, DocValuesEnum.NO_MORE_DOCS, vE_2_merged.nextDoc());
+
+ // close resources
+ r_1.close();
+ r_2.close();
+ merged.close();
+ w_1.close();
+ w_2.close();
+ w.close();
+ d_1.close();
+ d_2.close();
+ target.close();
+ }
+
private IndexWriterConfig writerConfig(boolean useCompoundFile) {
final IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer());
@@ -262,7 +332,8 @@ public class TestDocValuesIndexing exten
}
assertEquals("advance failed at index: " + i + " of " + r.numDocs()
+ " docs base:" + base, i, floatEnum.advance(i));
- assertEquals(floatEnum.getClass() + " index " + i, 2.0 * expected, enumRef.get(), 0.00001);
+ assertEquals(floatEnum.getClass() + " index " + i, 2.0 * expected,
+ enumRef.get(), 0.00001);
assertEquals("index " + i, 2.0 * expected, floats.getFloat(i),
0.00001);
}
@@ -365,7 +436,8 @@ public class TestDocValuesIndexing exten
.advance(i));
}
for (int j = 0; j < br.length; j++, upto++) {
- assertTrue(bytesEnum.getClass() + " enumRef not initialized " + msg, enumRef.bytes.length > 0);
+ assertTrue(" enumRef not initialized " + msg,
+ enumRef.bytes.length > 0);
assertEquals(
"EnumRef Byte at index " + j + " doesn't match - " + msg, upto,
enumRef.bytes[enumRef.offset + j]);
@@ -393,6 +465,7 @@ public class TestDocValuesIndexing exten
private DocValues getDocValues(IndexReader reader, String field)
throws IOException {
boolean optimized = reader.isOptimized();
+ reader.isCurrent();
Fields fields = optimized ? reader.getSequentialSubReaders()[0].fields()
: MultiFields.getFields(reader);
switch (random.nextInt(optimized ? 3 : 2)) { // case 2 only if optimized
@@ -438,9 +511,8 @@ public class TestDocValuesIndexing exten
}
private static EnumSet<Type> BYTES = EnumSet.of(Type.BYTES_FIXED_DEREF,
- Type.BYTES_FIXED_SORTED, Type.BYTES_FIXED_STRAIGHT,
- Type.BYTES_VAR_DEREF, Type.BYTES_VAR_SORTED,
- Type.BYTES_VAR_STRAIGHT);
+ Type.BYTES_FIXED_SORTED, Type.BYTES_FIXED_STRAIGHT, Type.BYTES_VAR_DEREF,
+ Type.BYTES_VAR_SORTED, Type.BYTES_VAR_STRAIGHT);
private static EnumSet<Type> NUMERICS = EnumSet.of(Type.PACKED_INTS,
Type.SIMPLE_FLOAT_4BYTE, Type.SIMPLE_FLOAT_8BYTE);
@@ -519,5 +591,4 @@ public class TestDocValuesIndexing exten
w.optimize();
return deleted;
}
-
}