You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/10/21 20:58:44 UTC
svn commit: r1534320 [18/39] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/lucene/expressions/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/
dev-tools/maven/lucene/ dev-tools/maven/lucene/expressions/...
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Mon Oct 21 18:58:24 2013
@@ -29,7 +29,6 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FailOnNonBulkMergesInfoStream;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import org.junit.Test;
public class TestConsistentFieldNumbers extends LuceneTestCase {
@@ -67,8 +66,8 @@ public class TestConsistentFieldNumbers
sis.read(dir);
assertEquals(2, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
@@ -85,7 +84,7 @@ public class TestConsistentFieldNumbers
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis3 = _TestUtil.getFieldInfos(sis.info(0).info);
+ FieldInfos fis3 = SegmentReader.readFieldInfos(sis.info(0));
assertEquals("f1", fis3.fieldInfo(0).name);
assertEquals("f2", fis3.fieldInfo(1).name);
@@ -130,8 +129,8 @@ public class TestConsistentFieldNumbers
sis.read(dir1);
assertEquals(2, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
@@ -161,7 +160,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
}
@@ -180,8 +179,8 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(2, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f1", fis2.fieldInfo(0).name);
@@ -203,9 +202,9 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(3, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
- FieldInfos fis3 = _TestUtil.getFieldInfos(sis.info(2).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
+ FieldInfos fis3 = SegmentReader.readFieldInfos(sis.info(2));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f1", fis2.fieldInfo(0).name);
@@ -237,7 +236,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f3", fis1.fieldInfo(2).name);
@@ -275,7 +274,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
for (SegmentInfoPerCommit si : sis) {
- FieldInfos fis = _TestUtil.getFieldInfos(si.info);
+ FieldInfos fis = SegmentReader.readFieldInfos(si);
for (FieldInfo fi : fis) {
Field expected = getField(Integer.parseInt(fi.name));
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestCrashCausesCorruptIndex.java Mon Oct 21 18:58:24 2013
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.BaseDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
@@ -146,7 +147,7 @@ public class TestCrashCausesCorruptIndex
* This test class provides direct access to "simulating" a crash right after
* realDirectory.createOutput(..) has been called on a certain specified name.
*/
- private static class CrashAfterCreateOutput extends Directory {
+ private static class CrashAfterCreateOutput extends BaseDirectory {
private Directory realDirectory;
private String crashAfterCreateOutput;
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Mon Oct 21 18:58:24 2013
@@ -1094,7 +1094,6 @@ public void testFilesOpenClose() throws
File tempDir = _TestUtil.getTempDir("testIndexExistsOnNonExistentDirectory");
tempDir.delete();
Directory dir = newFSDirectory(tempDir);
- System.out.println("dir=" + dir);
assertFalse(DirectoryReader.indexExists(dir));
dir.close();
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Mon Oct 21 18:58:24 2013
@@ -217,7 +217,7 @@ public class TestDoc extends LuceneTestC
final Codec codec = Codec.getDefault();
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
- final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, false, codec, null, null);
+ final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, false, codec, null);
SegmentMerger merger = new SegmentMerger(Arrays.<AtomicReader>asList(r1, r2),
si, InfoStream.getDefault(), trackingDir,
@@ -228,7 +228,7 @@ public class TestDoc extends LuceneTestC
r2.close();
final SegmentInfo info = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged,
si1.info.getDocCount() + si2.info.getDocCount(),
- false, codec, null, null);
+ false, codec, null);
info.setFiles(new HashSet<String>(trackingDir.getCreatedFiles()));
if (useCompoundFile) {
@@ -239,7 +239,7 @@ public class TestDoc extends LuceneTestC
}
}
- return new SegmentInfoPerCommit(info, 0, -1L);
+ return new SegmentInfoPerCommit(info, 0, -1L, -1L);
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java Mon Oct 21 18:58:24 2013
@@ -172,7 +172,7 @@ public class TestDocTermOrds extends Luc
AtomicReader slowR = SlowCompositeReaderWrapper.wrap(r);
verify(slowR, idToOrds, termsArray, null);
- FieldCache.DEFAULT.purge(slowR);
+ FieldCache.DEFAULT.purgeByCacheKey(slowR.getCoreCacheKey());
r.close();
dir.close();
@@ -291,7 +291,7 @@ public class TestDocTermOrds extends Luc
verify(slowR, idToOrdsPrefix, termsArray, prefixRef);
}
- FieldCache.DEFAULT.purge(slowR);
+ FieldCache.DEFAULT.purgeByCacheKey(slowR.getCoreCacheKey());
r.close();
dir.close();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java Mon Oct 21 18:58:24 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.util._TestUtil;
/** Tests the codec configuration defined by LuceneTestCase randomly
* (typically a mix across different fields).
@@ -28,4 +29,9 @@ public class TestDocValuesFormat extends
protected Codec getCodec() {
return Codec.getDefault();
}
+
+ @Override
+ protected boolean codecAcceptsHugeBinaryValues(String field) {
+ return _TestUtil.fieldSupportsHugeBinaryDocValues(field);
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Mon Oct 21 18:58:24 2013
@@ -26,9 +26,11 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
@@ -180,7 +182,7 @@ public class TestDocValuesIndexing exten
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = w.getReader();
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(getOnlySegmentReader(r), "field");
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(getOnlySegmentReader(r), "field", false);
BytesRef bytes1 = new BytesRef();
s.get(0, bytes1);
@@ -209,7 +211,7 @@ public class TestDocValuesIndexing exten
writer.addDocument(doc);
}
DirectoryReader r = writer.getReader();
- SlowCompositeReaderWrapper slow = new SlowCompositeReaderWrapper(r);
+ AtomicReader slow = SlowCompositeReaderWrapper.wrap(r);
FieldInfos fi = slow.getFieldInfos();
FieldInfo dvInfo = fi.fieldInfo("dv");
assertTrue(dvInfo.hasDocValues());
@@ -326,31 +328,7 @@ public class TestDocValuesIndexing exten
iwriter.close();
directory.close();
}
-
- public void testTooLargeBytes() throws IOException {
- Analyzer analyzer = new MockAnalyzer(random());
-
- Directory directory = newDirectory();
- // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
- IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
- iwc.setMergePolicy(newLogMergePolicy());
- IndexWriter iwriter = new IndexWriter(directory, iwc);
- Document doc = new Document();
- byte bytes[] = new byte[100000];
- BytesRef b = new BytesRef(bytes);
- random().nextBytes(bytes);
- doc.add(new BinaryDocValuesField("dv", b));
- try {
- iwriter.addDocument(doc);
- fail("did not get expected exception");
- } catch (IllegalArgumentException expected) {
- // expected
- }
- iwriter.close();
- directory.close();
- }
-
public void testTooLargeSortedBytes() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
@@ -775,4 +753,30 @@ public class TestDocValuesIndexing exten
dir.close();
}
+ public void testSameFieldNameForPostingAndDocValue() throws Exception {
+ // LUCENE-5192: FieldInfos.Builder neglected to update
+ // globalFieldNumbers.docValuesType map if the field existed, resulting in
+ // potentially adding the same field with different DV types.
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ doc.add(new StringField("f", "mock-value", Store.NO));
+ doc.add(new NumericDocValuesField("f", 5));
+ writer.addDocument(doc);
+ writer.commit();
+
+ doc = new Document();
+ doc.add(new BinaryDocValuesField("f", new BytesRef("mock")));
+ try {
+ writer.addDocument(doc);
+ fail("should not have succeeded to add a field with different DV type than what already exists");
+ } catch (IllegalArgumentException e) {
+ writer.rollback();
+ }
+
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java Mon Oct 21 18:58:24 2013
@@ -79,7 +79,7 @@ public class TestDocValuesWithThreads ex
//NumericDocValues ndv = ar.getNumericDocValues("number");
FieldCache.Longs ndv = FieldCache.DEFAULT.getLongs(ar, "number", false);
//BinaryDocValues bdv = ar.getBinaryDocValues("bytes");
- BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes");
+ BinaryDocValues bdv = FieldCache.DEFAULT.getTerms(ar, "bytes", false);
SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
startingGun.await();
int iters = atLeast(1000);
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Mon Oct 21 18:58:24 2013
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -49,7 +50,7 @@ public class TestDuelingCodecs extends L
public void setUp() throws Exception {
super.setUp();
- // for now its SimpleText vs Lucene42(random postings format)
+ // for now its SimpleText vs Lucene46(random postings format)
// as this gives the best overall coverage. when we have more
// codecs we should probably pick 2 from Codec.availableCodecs()
@@ -134,6 +135,11 @@ public class TestDuelingCodecs extends L
for (String trash : split) {
document.add(new SortedSetDocValuesField("sortedset", new BytesRef(trash)));
}
+ // add a numeric dv field sometimes
+ document.removeFields("sparsenumeric");
+ if (random.nextInt(4) == 2) {
+ document.add(new NumericDocValuesField("sparsenumeric", random.nextInt()));
+ }
writer.addDocument(document);
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java Mon Oct 21 18:58:24 2013
@@ -17,19 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
+import java.io.IOException;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
-
-import java.io.IOException;
-import java.util.Arrays;
+import org.apache.lucene.util.LuceneTestCase;
//import org.cnlp.utils.properties.ResourceBundleHelper;
@@ -60,14 +57,14 @@ public class TestFieldInfos extends Luce
//Use a RAMOutputStream
FieldInfosWriter writer = Codec.getDefault().fieldInfosFormat().getFieldInfosWriter();
- writer.write(dir, filename, fieldInfos, IOContext.DEFAULT);
+ writer.write(dir, filename, "", fieldInfos, IOContext.DEFAULT);
output.close();
return fieldInfos;
}
public FieldInfos readFieldInfos(Directory dir, String filename) throws IOException {
FieldInfosReader reader = Codec.getDefault().fieldInfosFormat().getFieldInfosReader();
- return reader.read(dir, filename, IOContext.DEFAULT);
+ return reader.read(dir, filename, "", IOContext.DEFAULT);
}
public void test() throws IOException {
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java Mon Oct 21 18:58:24 2013
@@ -36,6 +36,7 @@ import org.apache.lucene.document.String
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.store.BaseDirectory;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -112,7 +113,7 @@ public class TestFieldsReader extends Lu
}
- public static class FaultyFSDirectory extends Directory {
+ public static class FaultyFSDirectory extends BaseDirectory {
Directory fsDir;
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestFlushByRamOrCountsPolicy.java Mon Oct 21 18:58:24 2013
@@ -281,7 +281,10 @@ public class TestFlushByRamOrCountsPolic
Iterator<ThreadState> allActiveThreads = flushControl.allActiveThreadStates();
long bytesUsed = 0;
while (allActiveThreads.hasNext()) {
- bytesUsed += allActiveThreads.next().dwpt.bytesUsed();
+ ThreadState next = allActiveThreads.next();
+ if (next.dwpt != null) {
+ bytesUsed += next.dwpt.bytesUsed();
+ }
}
assertEquals(bytesUsed, flushControl.activeBytes());
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Mon Oct 21 18:58:24 2013
@@ -49,6 +49,8 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
@@ -68,8 +70,12 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SetOnce;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.BasicAutomata;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.packed.PackedInts;
import org.junit.Test;
@@ -1019,13 +1025,23 @@ public class TestIndexWriter extends Luc
@Override
public void run() {
// LUCENE-2239: won't work with NIOFS/MMAP
- Directory dir = new MockDirectoryWrapper(random, new RAMDirectory());
+ MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory());
+
+ // When interrupt arrives in w.close(), when it's
+ // writing liveDocs, this can lead to double-write of
+ // _X_N.del:
+ //dir.setPreventDoubleWrite(false);
IndexWriter w = null;
while(!finish) {
try {
while(!finish) {
if (w != null) {
+ // If interrupt arrives inside here, it's
+ // fine: we will cycle back and the first
+ // thing we do is try to close again,
+ // i.e. we'll never try to open a new writer
+ // until this one successfully closes:
w.close();
w = null;
}
@@ -1593,14 +1609,15 @@ public class TestIndexWriter extends Luc
@Override
public void reset() throws IOException {
- this.upto = 0;
- final StringBuilder b = new StringBuilder();
- final char[] buffer = new char[1024];
- int n;
- while ((n = input.read(buffer)) != -1) {
- b.append(buffer, 0, n);
- }
- this.tokens = b.toString().split(" ");
+ super.reset();
+ this.upto = 0;
+ final StringBuilder b = new StringBuilder();
+ final char[] buffer = new char[1024];
+ int n;
+ while ((n = input.read(buffer)) != -1) {
+ b.append(buffer, 0, n);
+ }
+ this.tokens = b.toString().split(" ");
}
}
@@ -1700,7 +1717,6 @@ public class TestIndexWriter extends Luc
w.deleteAll();
w.commit();
-
// Make sure we accumulate no files except for empty
// segments_N and segments.gen:
assertTrue(d.listAll().length <= 2);
@@ -1898,6 +1914,65 @@ public class TestIndexWriter extends Luc
}
}
+ // LUCENE-3849
+ public void testStopwordsPosIncHole() throws Exception {
+ Directory dir = newDirectory();
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader);
+ TokenStream stream = new MockTokenFilter(tokenizer, MockTokenFilter.ENGLISH_STOPSET);
+ return new TokenStreamComponents(tokenizer, stream);
+ }
+ };
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, a);
+ Document doc = new Document();
+ doc.add(new TextField("body", "just a", Field.Store.NO));
+ doc.add(new TextField("body", "test of gaps", Field.Store.NO));
+ iw.addDocument(doc);
+ IndexReader ir = iw.getReader();
+ iw.close();
+ IndexSearcher is = newSearcher(ir);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("body", "just"), 0);
+ pq.add(new Term("body", "test"), 2);
+ // body:"just ? test"
+ assertEquals(1, is.search(pq, 5).totalHits);
+ ir.close();
+ dir.close();
+ }
+
+ // LUCENE-3849
+ public void testStopwordsPosIncHole2() throws Exception {
+ // use two stopfilters for testing here
+ Directory dir = newDirectory();
+ final Automaton secondSet = BasicAutomata.makeString("foobar");
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader);
+ TokenStream stream = new MockTokenFilter(tokenizer, MockTokenFilter.ENGLISH_STOPSET);
+ stream = new MockTokenFilter(stream, new CharacterRunAutomaton(secondSet));
+ return new TokenStreamComponents(tokenizer, stream);
+ }
+ };
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, a);
+ Document doc = new Document();
+ doc.add(new TextField("body", "just a foobar", Field.Store.NO));
+ doc.add(new TextField("body", "test of gaps", Field.Store.NO));
+ iw.addDocument(doc);
+ IndexReader ir = iw.getReader();
+ iw.close();
+ IndexSearcher is = newSearcher(ir);
+ PhraseQuery pq = new PhraseQuery();
+ pq.add(new Term("body", "just"), 0);
+ pq.add(new Term("body", "test"), 3);
+ // body:"just ? ? test"
+ assertEquals(1, is.search(pq, 5).totalHits);
+ ir.close();
+ dir.close();
+ }
+
// here we do better, there is no current segments file, so we don't delete anything.
// however, if you actually go and make a commit, the next time you run indexwriter
// this file will be gone.
@@ -2174,10 +2249,36 @@ public class TestIndexWriter extends Luc
doc.add(newTextField("myfield", "a b c", Field.Store.NO));
writer.addDocument(doc);
assertTrue(writer.hasUncommittedChanges());
+
+ // Must commit, waitForMerges, commit again, to be
+ // certain that hasUncommittedChanges returns false:
+ writer.commit();
+ writer.waitForMerges();
writer.commit();
assertFalse(writer.hasUncommittedChanges());
writer.addDocument(doc);
assertTrue(writer.hasUncommittedChanges());
+ writer.commit();
+ doc = new Document();
+ doc.add(newStringField("id", "xyz", Field.Store.YES));
+ writer.addDocument(doc);
+ assertTrue(writer.hasUncommittedChanges());
+
+ // Must commit, waitForMerges, commit again, to be
+ // certain that hasUncommittedChanges returns false:
+ writer.commit();
+ writer.waitForMerges();
+ writer.commit();
+ assertFalse(writer.hasUncommittedChanges());
+ writer.deleteDocuments(new Term("id", "xyz"));
+ assertTrue(writer.hasUncommittedChanges());
+
+ // Must commit, waitForMerges, commit again, to be
+ // certain that hasUncommittedChanges returns false:
+ writer.commit();
+ writer.waitForMerges();
+ writer.commit();
+ assertFalse(writer.hasUncommittedChanges());
writer.close();
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -2188,4 +2289,55 @@ public class TestIndexWriter extends Luc
writer.close();
dir.close();
}
+
+ public void testMergeAllDeleted() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ final SetOnce<IndexWriter> iwRef = new SetOnce<IndexWriter>();
+ iwc.setInfoStream(new RandomIndexWriter.TestPointInfoStream(iwc.getInfoStream(), new RandomIndexWriter.TestPoint() {
+ @Override
+ public void apply(String message) {
+ if ("startCommitMerge".equals(message)) {
+ iwRef.get().setKeepFullyDeletedSegments(false);
+ } else if ("startMergeInit".equals(message)) {
+ iwRef.get().setKeepFullyDeletedSegments(true);
+ }
+ }
+ }));
+ IndexWriter evilWriter = new IndexWriter(dir, iwc);
+ iwRef.set(evilWriter);
+ for (int i = 0; i < 1000; i++) {
+ addDoc(evilWriter);
+ if (random().nextInt(17) == 0) {
+ evilWriter.commit();
+ }
+ }
+ evilWriter.deleteDocuments(new MatchAllDocsQuery());
+ evilWriter.forceMerge(1);
+ evilWriter.close();
+ dir.close();
+ }
+
+ // LUCENE-5239
+ public void testDeleteSameTermAcrossFields() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new TextField("a", "foo", Field.Store.NO));
+ w.addDocument(doc);
+
+ // Should not delete the document; with LUCENE-5239 the
+ // "foo" from the 2nd delete term would incorrectly
+ // match field a's "foo":
+ w.deleteDocuments(new Term("a", "xxx"));
+ w.deleteDocuments(new Term("b", "foo"));
+ IndexReader r = w.getReader();
+ w.close();
+
+ // Make sure document was not (incorrectly) deleted:
+ assertEquals(1, r.numDocs());
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Mon Oct 21 18:58:24 2013
@@ -30,7 +30,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -322,7 +324,7 @@ public class TestIndexWriterDelete exten
int value = 100;
try {
latch.await();
- for (int i = 0; i < 1000; i++) {
+ for (int j = 0; j < 1000; j++) {
Document doc = new Document();
doc.add(newTextField("content", "aaa", Field.Store.NO));
doc.add(newStringField("id", String.valueOf(id++), Field.Store.YES));
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon Oct 21 18:58:24 2013
@@ -28,7 +28,13 @@ import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -48,6 +54,7 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -381,7 +388,7 @@ public class TestIndexWriterExceptions e
doc.add(newTextField("field", "a field", Field.Store.YES));
w.addDocument(doc);
- Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) {
+ Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -537,21 +544,15 @@ public class TestIndexWriterExceptions e
public void eval(MockDirectoryWrapper dir) throws IOException {
if (doFail) {
StackTraceElement[] trace = new Exception().getStackTrace();
- boolean sawAppend = false;
boolean sawFlush = false;
for (int i = 0; i < trace.length; i++) {
- if (sawAppend && sawFlush) {
- break;
- }
- if (FreqProxTermsWriterPerField.class.getName().equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName())) {
- sawAppend = true;
- }
if ("flush".equals(trace[i].getMethodName())) {
sawFlush = true;
+ break;
}
}
- if (sawAppend && sawFlush && count++ >= 30) {
+ if (sawFlush && count++ >= 30) {
doFail = false;
throw new IOException("now failing during flush");
}
@@ -590,7 +591,7 @@ public class TestIndexWriterExceptions e
}
public void testDocumentsWriterExceptions() throws IOException {
- Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) {
+ Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -685,7 +686,7 @@ public class TestIndexWriterExceptions e
}
public void testDocumentsWriterExceptionThreads() throws Exception {
- Analyzer analyzer = new Analyzer(new Analyzer.PerFieldReuseStrategy()) {
+ Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -1690,7 +1691,226 @@ public class TestIndexWriterExceptions e
dir.close();
}
-
-
+
+ // Make sure if we hit a transient IOException (e.g., disk
+ // full), and then the exception stops (e.g., disk frees
+ // up), so we successfully close IW or open an NRT
+ // reader, we don't lose any deletes or updates:
+ public void testNoLostDeletesOrUpdates() throws Exception {
+ int deleteCount = 0;
+ int docBase = 0;
+ int docCount = 0;
+
+ MockDirectoryWrapper dir = newMockDirectory();
+ final AtomicBoolean shouldFail = new AtomicBoolean();
+ dir.failOn(new MockDirectoryWrapper.Failure() {
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ if (shouldFail.get() == false) {
+ return;
+ }
+
+ boolean sawSeal = false;
+ boolean sawWrite = false;
+ for (int i = 0; i < trace.length; i++) {
+ if ("sealFlushedSegment".equals(trace[i].getMethodName())) {
+ sawSeal = true;
+ break;
+ }
+ if ("writeLiveDocs".equals(trace[i].getMethodName()) || "writeFieldUpdates".equals(trace[i].getMethodName())) {
+ sawWrite = true;
+ }
+ }
+
+ // Don't throw exc if we are "flushing", else
+ // the segment is aborted and docs are lost:
+ if (sawWrite && sawSeal == false && random().nextInt(3) == 2) {
+ // Only sometimes throw the exc, so we get
+ // it sometimes on creating the file, on
+ // flushing buffer, on closing the file:
+ if (VERBOSE) {
+ System.out.println("TEST: now fail; thread=" + Thread.currentThread().getName() + " exc:");
+ new Throwable().printStackTrace(System.out);
+ }
+ shouldFail.set(false);
+ throw new FakeIOException();
+ }
+ }
+ });
+
+ RandomIndexWriter w = null;
+
+ for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {
+ int numDocs = atLeast(100);
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter + " numDocs=" + numDocs + " docBase=" + docBase + " delCount=" + deleteCount);
+ }
+ if (w == null) {
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ final MergeScheduler ms = iwc.getMergeScheduler();
+ if (ms instanceof ConcurrentMergeScheduler) {
+ final ConcurrentMergeScheduler suppressFakeIOE = new ConcurrentMergeScheduler() {
+ @Override
+ protected void handleMergeException(Throwable exc) {
+ // suppress only FakeIOException:
+ if (!(exc instanceof FakeIOException)) {
+ super.handleMergeException(exc);
+ }
+ }
+ };
+ final ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) ms;
+ suppressFakeIOE.setMaxMergesAndThreads(cms.getMaxMergeCount(), cms.getMaxThreadCount());
+ suppressFakeIOE.setMergeThreadPriority(cms.getMergeThreadPriority());
+ iwc.setMergeScheduler(suppressFakeIOE);
+ }
+
+ w = new RandomIndexWriter(random(), dir, iwc);
+ // Since we hit exc during merging, a partial
+ // forceMerge can easily return when there are still
+ // too many segments in the index:
+ w.setDoRandomForceMergeAssert(false);
+ }
+ for(int i=0;i<numDocs;i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", ""+(docBase+i), Field.Store.NO));
+ doc.add(new NumericDocValuesField("f", 1L));
+ doc.add(new NumericDocValuesField("cf", 2L));
+ w.addDocument(doc);
+ }
+ docCount += numDocs;
+
+ // TODO: we could make the test more evil, by letting
+ // it throw more than one exc, randomly, before "recovering"
+
+ // TODO: we could also install an infoStream and try
+ // to fail in "more evil" places inside BDS
+
+ shouldFail.set(true);
+ boolean doClose = false;
+
+ try {
+
+ boolean defaultCodecSupportsFieldUpdates = defaultCodecSupportsFieldUpdates();
+ for(int i=0;i<numDocs;i++) {
+ if (random().nextInt(10) == 7) {
+ boolean fieldUpdate = defaultCodecSupportsFieldUpdates && random().nextBoolean();
+ if (fieldUpdate) {
+ long value = iter;
+ if (VERBOSE) {
+ System.out.println(" update id=" + (docBase+i) + " to value " + value);
+ }
+ w.updateNumericDocValue(new Term("id", Integer.toString(docBase + i)), "f", value);
+ w.updateNumericDocValue(new Term("id", Integer.toString(docBase + i)), "cf", value * 2);
+ }
+
+ // sometimes do both deletes and updates
+ if (!fieldUpdate || random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println(" delete id=" + (docBase+i));
+ }
+ deleteCount++;
+ w.deleteDocuments(new Term("id", ""+(docBase+i)));
+ }
+ }
+ }
+
+ // Trigger writeLiveDocs so we hit fake exc:
+ IndexReader r = w.getReader(true);
+
+ // Sometimes we will make it here (we only randomly
+ // throw the exc):
+ assertEquals(docCount-deleteCount, r.numDocs());
+ r.close();
+
+ // Sometimes close, so the disk full happens on close:
+ if (random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println(" now close writer");
+ }
+ doClose = true;
+ w.close();
+ w = null;
+ }
+
+ } catch (IOException ioe) {
+ // FakeIOException can be thrown from mergeMiddle, in which case IW
+ // registers it before our CMS gets to suppress it. IW.forceMerge later
+ // throws it as a wrapped IOE, so don't fail in this case.
+ if (ioe instanceof FakeIOException || (ioe.getCause() != null && ioe.getCause() instanceof FakeIOException)) {
+ // expected
+ if (VERBOSE) {
+ System.out.println("TEST: w.close() hit expected IOE");
+ }
+ } else {
+ throw ioe;
+ }
+ }
+ shouldFail.set(false);
+
+ IndexReader r;
+
+ if (doClose && w != null) {
+ if (VERBOSE) {
+ System.out.println(" now 2nd close writer");
+ }
+ w.close();
+ w = null;
+ }
+
+ if (w == null || random().nextBoolean()) {
+ // Open non-NRT reader, to make sure the "on
+ // disk" bits are good:
+ if (VERBOSE) {
+ System.out.println("TEST: verify against non-NRT reader");
+ }
+ if (w != null) {
+ w.commit();
+ }
+ r = DirectoryReader.open(dir);
+ } else {
+ if (VERBOSE) {
+ System.out.println("TEST: verify against NRT reader");
+ }
+ r = w.getReader();
+ }
+ assertEquals(docCount-deleteCount, r.numDocs());
+ for (AtomicReaderContext context : r.leaves()) {
+ Bits liveDocs = context.reader().getLiveDocs();
+ NumericDocValues f = context.reader().getNumericDocValues("f");
+ NumericDocValues cf = context.reader().getNumericDocValues("cf");
+ for (int i = 0; i < context.reader().maxDoc(); i++) {
+ if (liveDocs == null || liveDocs.get(i)) {
+ assertEquals("doc=" + (docBase + i), cf.get(i), f.get(i) * 2);
+ }
+ }
+ }
+
+ r.close();
+
+ // Sometimes re-use RIW, other times open new one:
+ if (w != null && random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println("TEST: close writer");
+ }
+ w.close();
+ w = null;
+ }
+
+ docBase += numDocs;
+ }
+
+ if (w != null) {
+ w.close();
+ }
+
+ // Final verify:
+ IndexReader r = DirectoryReader.open(dir);
+ assertEquals(docCount-deleteCount, r.numDocs());
+ r.close();
+
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java Mon Oct 21 18:58:24 2013
@@ -83,7 +83,7 @@ public class TestIndexWriterForceMerge e
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMaxBufferedDocs(2).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
-
+
for(int iter=0;iter<10;iter++) {
for(int i=0;i<19;i++)
writer.addDocument(doc);
@@ -96,7 +96,6 @@ public class TestIndexWriterForceMerge e
sis.read(dir);
final int segCount = sis.size();
-
writer.forceMerge(7);
writer.commit();
writer.waitForMerges();
@@ -108,7 +107,7 @@ public class TestIndexWriterForceMerge e
if (segCount < 7)
assertEquals(segCount, optSegCount);
else
- assertEquals(7, optSegCount);
+ assertEquals("seg: " + segCount, 7, optSegCount);
}
writer.close();
dir.close();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Mon Oct 21 18:58:24 2013
@@ -490,7 +490,8 @@ public class TestIndexWriterOnDiskFull e
setReaderPooling(true).
setMergePolicy(newLogMergePolicy(2))
);
- _TestUtil.keepFullyDeletedSegments(w);
+ // we can do this because we add/delete/add (and dont merge to "nothing")
+ w.setKeepFullyDeletedSegments(true);
Document doc = new Document();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Mon Oct 21 18:58:24 2013
@@ -28,7 +28,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -36,6 +35,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
@@ -43,6 +43,7 @@ import org.apache.lucene.util.InfoStream
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util._TestUtil;
+import org.junit.Test;
public class TestIndexWriterReader extends LuceneTestCase {
@@ -157,6 +158,7 @@ public class TestIndexWriterReader exten
assertEquals(1, count(new Term("id", Integer.toString(8000)), r2));
r1.close();
+ assertTrue(r2.isCurrent());
writer.close();
assertTrue(r2.isCurrent());
@@ -1041,4 +1043,60 @@ public class TestIndexWriterReader exten
w.close();
d.close();
}
+
+ @Test
+ public void testNRTOpenExceptions() throws Exception {
+ // LUCENE-5262: test that several failed attempts to obtain an NRT reader
+ // don't leak file handles.
+ MockDirectoryWrapper dir = newMockDirectory();
+ final AtomicBoolean shouldFail = new AtomicBoolean();
+ dir.failOn(new MockDirectoryWrapper.Failure() {
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ if (shouldFail.get()) {
+ for (int i = 0; i < trace.length; i++) {
+ if ("getReadOnlyClone".equals(trace[i].getMethodName())) {
+ if (VERBOSE) {
+ System.out.println("TEST: now fail; exc:");
+ new Throwable().printStackTrace(System.out);
+ }
+ shouldFail.set(false);
+ throw new FakeIOException();
+ }
+ }
+ }
+ }
+ });
+
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges from getting in the way
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ // create a segment and open an NRT reader
+ writer.addDocument(new Document());
+ writer.getReader().close();
+
+ // add a new document so a new NRT reader is required
+ writer.addDocument(new Document());
+
+ // try to obtain an NRT reader twice: first time it fails and closes all the
+ // other NRT readers. second time it fails, but also fails to close the
+ // other NRT reader, since it is already marked closed!
+ for (int i = 0; i < 2; i++) {
+ shouldFail.set(true);
+ try {
+ writer.getReader().close();
+ } catch (FakeIOException e) {
+ // expected
+ if (VERBOSE) {
+ System.out.println("hit expected fake IOE");
+ }
+ }
+ }
+
+ writer.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Mon Oct 21 18:58:24 2013
@@ -177,6 +177,9 @@ public class TestIndexWriterWithThreads
int NUM_THREADS = 3;
int numIterations = TEST_NIGHTLY ? 7 : 3;
for(int iter=0;iter<numIterations;iter++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter);
+ }
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(
@@ -209,6 +212,9 @@ public class TestIndexWriterWithThreads
}
}
+ if (VERBOSE) {
+ System.out.println("\nTEST: now close");
+ }
writer.close(false);
// Make sure threads that are adding docs are not hung:
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java Mon Oct 21 18:58:24 2013
@@ -47,30 +47,29 @@ public class TestLongPostings extends Lu
if (other != null && s.equals(other)) {
continue;
}
- final TokenStream ts = a.tokenStream("foo", s);
- final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
- final BytesRef termBytes = termAtt.getBytesRef();
- ts.reset();
-
- int count = 0;
- boolean changed = false;
-
- while(ts.incrementToken()) {
- termAtt.fillBytesRef();
- if (count == 0 && !termBytes.utf8ToString().equals(s)) {
- // The value was changed during analysis. Keep iterating so the
- // tokenStream is exhausted.
- changed = true;
+ try (TokenStream ts = a.tokenStream("foo", s)) {
+ final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
+ final BytesRef termBytes = termAtt.getBytesRef();
+ ts.reset();
+
+ int count = 0;
+ boolean changed = false;
+
+ while(ts.incrementToken()) {
+ termAtt.fillBytesRef();
+ if (count == 0 && !termBytes.utf8ToString().equals(s)) {
+ // The value was changed during analysis. Keep iterating so the
+ // tokenStream is exhausted.
+ changed = true;
+ }
+ count++;
}
- count++;
- }
-
- ts.end();
- ts.close();
- // Did we iterate just once and the value was unchanged?
- if (!changed && count == 1) {
- return s;
+ ts.end();
+ // Did we iterate just once and the value was unchanged?
+ if (!changed && count == 1) {
+ return s;
+ }
}
}
}
@@ -199,6 +198,9 @@ public class TestLongPostings extends Lu
}
if (random().nextInt(6) == 3) {
+ if (VERBOSE) {
+ System.out.println(" check positions");
+ }
final int freq = postings.freq();
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java Mon Oct 21 18:58:24 2013
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Numeri
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -120,6 +121,9 @@ public class TestMultiDocValues extends
int numDocs = atLeast(500);
for (int i = 0; i < numDocs; i++) {
ref.copyChars(_TestUtil.randomUnicodeString(random()));
+ if (defaultCodecSupportsDocsWithField() && random().nextInt(7) == 0) {
+ iw.addDocument(new Document());
+ }
iw.addDocument(doc);
if (random().nextInt(17) == 0) {
iw.commit();
@@ -139,7 +143,7 @@ public class TestMultiDocValues extends
for (int i = 0; i < numDocs; i++) {
// check ord
assertEquals(single.getOrd(i), multi.getOrd(i));
- // check ord value
+ // check value
single.get(i, expected);
multi.get(i, actual);
assertEquals(expected, actual);
@@ -321,4 +325,52 @@ public class TestMultiDocValues extends
ir2.close();
dir.close();
}
+
+ public void testDocsWithField() throws Exception {
+ assumeTrue("codec does not support docsWithField", defaultCodecSupportsDocsWithField());
+ Directory dir = newDirectory();
+
+ IndexWriterConfig iwc = newIndexWriterConfig(random(), TEST_VERSION_CURRENT, null);
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ int numDocs = atLeast(500);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ if (random().nextInt(4) >= 0) {
+ doc.add(new NumericDocValuesField("numbers", random().nextLong()));
+ }
+ doc.add(new NumericDocValuesField("numbersAlways", random().nextLong()));
+ iw.addDocument(doc);
+ if (random().nextInt(17) == 0) {
+ iw.commit();
+ }
+ }
+ DirectoryReader ir = iw.getReader();
+ iw.forceMerge(1);
+ DirectoryReader ir2 = iw.getReader();
+ AtomicReader merged = getOnlySegmentReader(ir2);
+ iw.close();
+
+ Bits multi = MultiDocValues.getDocsWithField(ir, "numbers");
+ Bits single = merged.getDocsWithField("numbers");
+ if (multi == null) {
+ assertNull(single);
+ } else {
+ assertEquals(single.length(), multi.length());
+ for (int i = 0; i < numDocs; i++) {
+ assertEquals(single.get(i), multi.get(i));
+ }
+ }
+
+ multi = MultiDocValues.getDocsWithField(ir, "numbersAlways");
+ single = merged.getDocsWithField("numbersAlways");
+ assertEquals(single.length(), multi.length());
+ for (int i = 0; i < numDocs; i++) {
+ assertEquals(single.get(i), multi.get(i));
+ }
+ ir.close();
+ ir2.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java Mon Oct 21 18:58:24 2013
@@ -37,7 +37,8 @@ public class TestMultiFields extends Luc
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
- _TestUtil.keepFullyDeletedSegments(w);
+ // we can do this because we use NoMergePolicy (and dont merge to "nothing")
+ w.setKeepFullyDeletedSegments(true);
Map<BytesRef,List<Integer>> docs = new HashMap<BytesRef,List<Integer>>();
Set<Integer> deleted = new HashSet<Integer>();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java Mon Oct 21 18:58:24 2013
@@ -363,11 +363,11 @@ public class TestPayloads extends Lucene
Map<String,PayloadData> fieldToData = new HashMap<String,PayloadData>();
public PayloadAnalyzer() {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
}
public PayloadAnalyzer(String field, byte[] data, int offset, int length) {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
setPayloadData(field, data, offset, length);
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Mon Oct 21 18:58:24 2013
@@ -379,7 +379,7 @@ public class TestPostingsOffsets extends
riw.addDocument(doc);
}
CompositeReader ir = riw.getReader();
- SlowCompositeReaderWrapper slow = new SlowCompositeReaderWrapper(ir);
+ AtomicReader slow = SlowCompositeReaderWrapper.wrap(ir);
FieldInfos fis = slow.getFieldInfos();
assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, fis.fieldInfo("foo").getIndexOptions());
slow.close();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java Mon Oct 21 18:58:24 2013
@@ -25,6 +25,7 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.MergedIterator;
import org.apache.lucene.util._TestUtil;
public class TestPrefixCodedTerms extends LuceneTestCase {
@@ -66,15 +67,6 @@ public class TestPrefixCodedTerms extend
}
assertFalse(expected.hasNext());
}
-
- @SuppressWarnings("unchecked")
- public void testMergeEmpty() {
- Iterator<Term> merged = new MergedIterator<Term>();
- assertFalse(merged.hasNext());
-
- merged = new MergedIterator<Term>(new PrefixCodedTerms.Builder().finish().iterator(), new PrefixCodedTerms.Builder().finish().iterator());
- assertFalse(merged.hasNext());
- }
@SuppressWarnings("unchecked")
public void testMergeOne() {
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Mon Oct 21 18:58:24 2013
@@ -78,7 +78,7 @@ public class TestSegmentMerger extends L
public void testMerge() throws IOException {
final Codec codec = Codec.getDefault();
- final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null, null);
+ final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null);
SegmentMerger merger = new SegmentMerger(Arrays.<AtomicReader>asList(reader1, reader2),
si, InfoStream.getDefault(), mergedDir,
@@ -89,8 +89,8 @@ public class TestSegmentMerger extends L
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = new SegmentReader(new SegmentInfoPerCommit(
new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, docsMerged,
- false, codec, null, null),
- 0, -1L),
+ false, codec, null),
+ 0, -1L, -1L),
newIOContext(random()));
assertTrue(mergedReader != null);
assertTrue(mergedReader.numDocs() == 2);
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java Mon Oct 21 18:58:24 2013
@@ -127,7 +127,7 @@ public class TestTermVectorsReader exten
seg = writer.newestSegment();
writer.close();
- fieldInfos = _TestUtil.getFieldInfos(seg.info);
+ fieldInfos = SegmentReader.readFieldInfos(seg);
}
@Override
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Mon Oct 21 18:58:24 2013
@@ -174,17 +174,18 @@ public class TestTermVectorsWriter exten
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TokenStream stream = analyzer.tokenStream("field", "abcd ");
- stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
- stream = new CachingTokenFilter(stream);
- FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
- Field f = new Field("field", stream, customType);
- doc.add(f);
- doc.add(f);
- w.addDocument(doc);
+ try (TokenStream stream = analyzer.tokenStream("field", "abcd ")) {
+ stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
+ TokenStream cachedStream = new CachingTokenFilter(stream);
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectorOffsets(true);
+ Field f = new Field("field", cachedStream, customType);
+ doc.add(f);
+ doc.add(f);
+ w.addDocument(doc);
+ }
w.close();
IndexReader r = DirectoryReader.open(dir);
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java Mon Oct 21 18:58:24 2013
@@ -340,7 +340,6 @@ public class TestTermsEnum extends Lucen
loc++;
} while (loc < termsArray.length && !acceptTermsSet.contains(termsArray[loc]));
}
-
assertNull(te.next());
}
}
@@ -771,4 +770,116 @@ public class TestTermsEnum extends Lucen
r.close();
dir.close();
}
+ public void testIntersectStartTerm() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwc.setMergePolicy(new LogDocMergePolicy());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("field", "abc", Field.Store.NO));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(newStringField("field", "abd", Field.Store.NO));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(newStringField("field", "acd", Field.Store.NO));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(newStringField("field", "bcd", Field.Store.NO));
+ w.addDocument(doc);
+
+ w.forceMerge(1);
+ DirectoryReader r = w.getReader();
+ w.close();
+ AtomicReader sub = getOnlySegmentReader(r);
+ Terms terms = sub.fields().terms("field");
+
+ Automaton automaton = new RegExp(".*d", RegExp.NONE).toAutomaton();
+ CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false);
+ TermsEnum te;
+
+ // should seek to startTerm
+ te = terms.intersect(ca, new BytesRef("aad"));
+ assertEquals("abd", te.next().utf8ToString());
+ assertEquals(1, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc());
+ assertEquals("acd", te.next().utf8ToString());
+ assertEquals(2, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc());
+ assertEquals("bcd", te.next().utf8ToString());
+ assertEquals(3, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc());
+ assertNull(te.next());
+
+ // should fail to find ceil label on second arc, rewind
+ te = terms.intersect(ca, new BytesRef("add"));
+ assertEquals("bcd", te.next().utf8ToString());
+ assertEquals(3, te.docs(null, null, DocsEnum.FLAG_NONE).nextDoc());
+ assertNull(te.next());
+
+ // should reach end
+ te = terms.intersect(ca, new BytesRef("bcd"));
+ assertNull(te.next());
+ te = terms.intersect(ca, new BytesRef("ddd"));
+ assertNull(te.next());
+
+ r.close();
+ dir.close();
+ }
+
+ public void testIntersectEmptyString() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwc.setMergePolicy(new LogDocMergePolicy());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("field", "", Field.Store.NO));
+ doc.add(newStringField("field", "abc", Field.Store.NO));
+ w.addDocument(doc);
+
+ doc = new Document();
+ // add empty string to both documents, so that singletonDocID == -1.
+ // For a FST-based term dict, we'll expect to see the first arc is
+ // flaged with HAS_FINAL_OUTPUT
+ doc.add(newStringField("field", "abc", Field.Store.NO));
+ doc.add(newStringField("field", "", Field.Store.NO));
+ w.addDocument(doc);
+
+ w.forceMerge(1);
+ DirectoryReader r = w.getReader();
+ w.close();
+ AtomicReader sub = getOnlySegmentReader(r);
+ Terms terms = sub.fields().terms("field");
+
+ Automaton automaton = new RegExp(".*", RegExp.NONE).toAutomaton(); // accept ALL
+ CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false);
+
+ TermsEnum te = terms.intersect(ca, null);
+ DocsEnum de;
+
+ assertEquals("", te.next().utf8ToString());
+ de = te.docs(null, null, DocsEnum.FLAG_NONE);
+ assertEquals(0, de.nextDoc());
+ assertEquals(1, de.nextDoc());
+
+ assertEquals("abc", te.next().utf8ToString());
+ de = te.docs(null, null, DocsEnum.FLAG_NONE);
+ assertEquals(0, de.nextDoc());
+ assertEquals(1, de.nextDoc());
+
+ assertNull(te.next());
+
+ // pass empty string
+ te = terms.intersect(ca, new BytesRef(""));
+
+ assertEquals("abc", te.next().utf8ToString());
+ de = te.docs(null, null, DocsEnum.FLAG_NONE);
+ assertEquals(0, de.nextDoc());
+ assertEquals(1, de.nextDoc());
+
+ assertNull(te.next());
+
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java Mon Oct 21 18:58:24 2013
@@ -38,6 +38,10 @@ public class TestTransactions extends Lu
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
if (TestTransactions.doFail && random().nextInt() % 10 <= 3) {
+ if (VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + " TEST: now fail on purpose");
+ new Throwable().printStackTrace(System.out);
+ }
throw new IOException("now failing randomly but on purpose");
}
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/index.41.cfs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/index.41.cfs.zip?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/index.41.nocfs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/index/index.41.nocfs.zip?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Mon Oct 21 18:58:24 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -32,11 +33,113 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
public class TestCachingWrapperFilter extends LuceneTestCase {
+ Directory dir;
+ DirectoryReader ir;
+ IndexSearcher is;
+ RandomIndexWriter iw;
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ dir = newDirectory();
+ iw = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ Field idField = new StringField("id", "", Field.Store.NO);
+ doc.add(idField);
+ // add 500 docs with id 0..499
+ for (int i = 0; i < 500; i++) {
+ idField.setStringValue(Integer.toString(i));
+ iw.addDocument(doc);
+ }
+ // delete 20 of them
+ for (int i = 0; i < 20; i++) {
+ iw.deleteDocuments(new Term("id", Integer.toString(random().nextInt(iw.maxDoc()))));
+ }
+ ir = iw.getReader();
+ is = newSearcher(ir);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(iw, ir, dir);
+ super.tearDown();
+ }
+
+ private void assertFilterEquals(Filter f1, Filter f2) throws Exception {
+ Query query = new MatchAllDocsQuery();
+ TopDocs hits1 = is.search(query, f1, ir.maxDoc());
+ TopDocs hits2 = is.search(query, f2, ir.maxDoc());
+ assertEquals(hits1.totalHits, hits2.totalHits);
+ CheckHits.checkEqual(query, hits1.scoreDocs, hits2.scoreDocs);
+ // now do it again to confirm caching works
+ TopDocs hits3 = is.search(query, f1, ir.maxDoc());
+ TopDocs hits4 = is.search(query, f2, ir.maxDoc());
+ assertEquals(hits3.totalHits, hits4.totalHits);
+ CheckHits.checkEqual(query, hits3.scoreDocs, hits4.scoreDocs);
+ }
+
+ /** test null iterator */
+ public void testEmpty() throws Exception {
+ Query query = new BooleanQuery();
+ Filter expected = new QueryWrapperFilter(query);
+ Filter actual = new CachingWrapperFilter(expected);
+ assertFilterEquals(expected, actual);
+ }
+
+ /** test iterator returns NO_MORE_DOCS */
+ public void testEmpty2() throws Exception {
+ BooleanQuery query = new BooleanQuery();
+ query.add(new TermQuery(new Term("id", "0")), BooleanClause.Occur.MUST);
+ query.add(new TermQuery(new Term("id", "0")), BooleanClause.Occur.MUST_NOT);
+ Filter expected = new QueryWrapperFilter(query);
+ Filter actual = new CachingWrapperFilter(expected);
+ assertFilterEquals(expected, actual);
+ }
+
+ /** test null docidset */
+ public void testEmpty3() throws Exception {
+ Filter expected = new PrefixFilter(new Term("bogusField", "bogusVal"));
+ Filter actual = new CachingWrapperFilter(expected);
+ assertFilterEquals(expected, actual);
+ }
+
+ /** test iterator returns single document */
+ public void testSingle() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ int id = random().nextInt(ir.maxDoc());
+ Query query = new TermQuery(new Term("id", Integer.toString(id)));
+ Filter expected = new QueryWrapperFilter(query);
+ Filter actual = new CachingWrapperFilter(expected);
+ assertFilterEquals(expected, actual);
+ }
+ }
+
+ /** test sparse filters (match single documents) */
+ public void testSparse() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ int id_start = random().nextInt(ir.maxDoc()-1);
+ int id_end = id_start + 1;
+ Query query = TermRangeQuery.newStringRange("id",
+ Integer.toString(id_start), Integer.toString(id_end), true, true);
+ Filter expected = new QueryWrapperFilter(query);
+ Filter actual = new CachingWrapperFilter(expected);
+ assertFilterEquals(expected, actual);
+ }
+ }
+
+ /** test dense filters (match entire index) */
+ public void testDense() throws Exception {
+ Query query = new MatchAllDocsQuery();
+ Filter expected = new QueryWrapperFilter(query);
+ Filter actual = new CachingWrapperFilter(expected);
+ assertFilterEquals(expected, actual);
+ }
+
public void testCachingWorks() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -173,7 +276,6 @@ public class TestCachingWrapperFilter ex
// asserts below requires no unexpected merges:
setMergePolicy(newLogMergePolicy(10))
);
- _TestUtil.keepFullyDeletedSegments(writer.w);
// NOTE: cannot use writer.getReader because RIW (on
// flipping a coin) may give us a newly opened reader,