You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/01/04 15:53:21 UTC
svn commit: r1649347 [7/31] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ dev-tools/idea/solr/contrib/dataimporthandler-extras/
dev-tools/idea/solr/contrib/extraction/
dev-tools/idea/solr/contrib/map-reduce/
dev-tools/idea/solr/contrib/velocity/ ...
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.compres
*/
import java.io.IOException;
+import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -27,14 +28,20 @@ import org.apache.lucene.index.BaseStore
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.Test;
+
import com.carrotsearch.randomizedtesting.generators.RandomInts;
public class TestCompressingStoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
+ static final long SECOND = 1000L;
+ static final long HOUR = 60 * 60 * SECOND;
+ static final long DAY = 24 * HOUR;
+
@Override
protected Codec getCodec() {
return CompressingCodec.randomInstance(random());
@@ -81,4 +88,176 @@ public class TestCompressingStoredFields
dir.close();
}
}
+
+ public void testZFloat() throws Exception {
+ byte buffer[] = new byte[5]; // we never need more than 5 bytes
+ ByteArrayDataOutput out = new ByteArrayDataOutput(buffer);
+ ByteArrayDataInput in = new ByteArrayDataInput(buffer);
+
+ // round-trip small integer values
+ for (int i = Short.MIN_VALUE; i < Short.MAX_VALUE; i++) {
+ float f = (float) i;
+ CompressingStoredFieldsWriter.writeZFloat(out, f);
+ in.reset(buffer, 0, out.getPosition());
+ float g = CompressingStoredFieldsReader.readZFloat(in);
+ assertTrue(in.eof());
+ assertEquals(Float.floatToIntBits(f), Float.floatToIntBits(g));
+
+ // check that compression actually works
+ if (i >= -1 && i <= 123) {
+ assertEquals(1, out.getPosition()); // single byte compression
+ }
+ out.reset(buffer);
+ }
+
+ // round-trip special values
+ float special[] = {
+ -0.0f,
+ +0.0f,
+ Float.NEGATIVE_INFINITY,
+ Float.POSITIVE_INFINITY,
+ Float.MIN_VALUE,
+ Float.MAX_VALUE,
+ Float.NaN,
+ };
+
+ for (float f : special) {
+ CompressingStoredFieldsWriter.writeZFloat(out, f);
+ in.reset(buffer, 0, out.getPosition());
+ float g = CompressingStoredFieldsReader.readZFloat(in);
+ assertTrue(in.eof());
+ assertEquals(Float.floatToIntBits(f), Float.floatToIntBits(g));
+ out.reset(buffer);
+ }
+
+ // round-trip random values
+ Random r = random();
+ for (int i = 0; i < 100000; i++) {
+ float f = r.nextFloat() * (random().nextInt(100) - 50);
+ CompressingStoredFieldsWriter.writeZFloat(out, f);
+ assertTrue("length=" + out.getPosition() + ", f=" + f, out.getPosition() <= ((Float.floatToIntBits(f) >>> 31) == 1 ? 5 : 4));
+ in.reset(buffer, 0, out.getPosition());
+ float g = CompressingStoredFieldsReader.readZFloat(in);
+ assertTrue(in.eof());
+ assertEquals(Float.floatToIntBits(f), Float.floatToIntBits(g));
+ out.reset(buffer);
+ }
+ }
+
+ public void testZDouble() throws Exception {
+ byte buffer[] = new byte[9]; // we never need more than 9 bytes
+ ByteArrayDataOutput out = new ByteArrayDataOutput(buffer);
+ ByteArrayDataInput in = new ByteArrayDataInput(buffer);
+
+ // round-trip small integer values
+ for (int i = Short.MIN_VALUE; i < Short.MAX_VALUE; i++) {
+ double x = (double) i;
+ CompressingStoredFieldsWriter.writeZDouble(out, x);
+ in.reset(buffer, 0, out.getPosition());
+ double y = CompressingStoredFieldsReader.readZDouble(in);
+ assertTrue(in.eof());
+ assertEquals(Double.doubleToLongBits(x), Double.doubleToLongBits(y));
+
+ // check that compression actually works
+ if (i >= -1 && i <= 124) {
+ assertEquals(1, out.getPosition()); // single byte compression
+ }
+ out.reset(buffer);
+ }
+
+ // round-trip special values
+ double special[] = {
+ -0.0d,
+ +0.0d,
+ Double.NEGATIVE_INFINITY,
+ Double.POSITIVE_INFINITY,
+ Double.MIN_VALUE,
+ Double.MAX_VALUE,
+ Double.NaN
+ };
+
+ for (double x : special) {
+ CompressingStoredFieldsWriter.writeZDouble(out, x);
+ in.reset(buffer, 0, out.getPosition());
+ double y = CompressingStoredFieldsReader.readZDouble(in);
+ assertTrue(in.eof());
+ assertEquals(Double.doubleToLongBits(x), Double.doubleToLongBits(y));
+ out.reset(buffer);
+ }
+
+ // round-trip random values
+ Random r = random();
+ for (int i = 0; i < 100000; i++) {
+ double x = r.nextDouble() * (random().nextInt(100) - 50);
+ CompressingStoredFieldsWriter.writeZDouble(out, x);
+ assertTrue("length=" + out.getPosition() + ", d=" + x, out.getPosition() <= (x < 0 ? 9 : 8));
+ in.reset(buffer, 0, out.getPosition());
+ double y = CompressingStoredFieldsReader.readZDouble(in);
+ assertTrue(in.eof());
+ assertEquals(Double.doubleToLongBits(x), Double.doubleToLongBits(y));
+ out.reset(buffer);
+ }
+
+ // same with floats
+ for (int i = 0; i < 100000; i++) {
+ double x = (double) (r.nextFloat() * (random().nextInt(100) - 50));
+ CompressingStoredFieldsWriter.writeZDouble(out, x);
+ assertTrue("length=" + out.getPosition() + ", d=" + x, out.getPosition() <= 5);
+ in.reset(buffer, 0, out.getPosition());
+ double y = CompressingStoredFieldsReader.readZDouble(in);
+ assertTrue(in.eof());
+ assertEquals(Double.doubleToLongBits(x), Double.doubleToLongBits(y));
+ out.reset(buffer);
+ }
+ }
+
+ public void testTLong() throws Exception {
+ byte buffer[] = new byte[10]; // we never need more than 10 bytes
+ ByteArrayDataOutput out = new ByteArrayDataOutput(buffer);
+ ByteArrayDataInput in = new ByteArrayDataInput(buffer);
+
+ // round-trip small integer values
+ for (int i = Short.MIN_VALUE; i < Short.MAX_VALUE; i++) {
+ for (long mul : new long[] {SECOND, HOUR, DAY}) {
+ long l1 = (long) i * mul;
+ CompressingStoredFieldsWriter.writeTLong(out, l1);
+ in.reset(buffer, 0, out.getPosition());
+ long l2 = CompressingStoredFieldsReader.readTLong(in);
+ assertTrue(in.eof());
+ assertEquals(l1, l2);
+
+ // check that compression actually works
+ if (i >= -16 && i <= 15) {
+ assertEquals(1, out.getPosition()); // single byte compression
+ }
+ out.reset(buffer);
+ }
+ }
+
+ // round-trip random values
+ Random r = random();
+ for (int i = 0; i < 100000; i++) {
+ final int numBits = r.nextInt(65);
+ long l1 = r.nextLong() & ((1L << numBits) - 1);
+ switch (r.nextInt(4)) {
+ case 0:
+ l1 *= SECOND;
+ break;
+ case 1:
+ l1 *= HOUR;
+ break;
+ case 2:
+ l1 *= DAY;
+ break;
+ default:
+ break;
+ }
+ CompressingStoredFieldsWriter.writeTLong(out, l1);
+ in.reset(buffer, 0, out.getPosition());
+ long l2 = CompressingStoredFieldsReader.readTLong(in);
+ assertTrue(in.eof());
+ assertEquals(l1, l2);
+ out.reset(buffer);
+ }
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java Sun Jan 4 14:53:12 2015
@@ -489,7 +489,7 @@ public class TestBlockPostingsFormat3 ex
assertEquals(freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(leftDocs.nextPosition(), rightDocs.nextPosition());
- // we don't compare the payloads, its allowed that one is empty etc
+ // we don't compare the payloads, it's allowed that one is empty etc
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java Sun Jan 4 14:53:12 2015
@@ -57,6 +57,11 @@ public class TestLucene50StoredFieldsFor
DirectoryReader ir = DirectoryReader.open(dir);
assertEquals(10, ir.numDocs());
+ for (int i = 0; i < 10; i++) {
+ Document doc = ir.document(i);
+ assertEquals("value1", doc.getString("field1"));
+ assertEquals("value2", doc.getString("field2"));
+ }
ir.close();
// checkindex
dir.close();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestAtomFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestAtomFields.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestAtomFields.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestAtomFields.java Sun Jan 4 14:53:12 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.document;
*/
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -383,7 +384,8 @@ public class TestAtomFields extends Luce
}
public void testMultiValuedSort() throws Exception {
- Directory dir = newDirectory();
+ assumeTrue("DV format not supported", Arrays.asList("Memory", "SimpleText").contains(TestUtil.getDocValuesFormat("field")) == false);
+
RandomIndexWriter w = newRandomIndexWriter(dir);
FieldTypes fieldTypes = w.getFieldTypes();
fieldTypes.setMultiValued("field");
@@ -423,6 +425,5 @@ public class TestAtomFields extends Luce
r.close();
w.close();
- dir.close();
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java Sun Jan 4 14:53:12 2015
@@ -55,13 +55,13 @@ public class TestBinaryDocument extends
Document docFromReader = reader.document(0);
assertTrue(docFromReader != null);
- /** fetch the binary stored field and compare it's content with the original one */
+ /** fetch the binary stored field and compare its content with the original one */
BytesRef bytes = docFromReader.getBinary("binaryStored");
assertNotNull(bytes);
String binaryFldStoredTest = new String(bytes.bytes, bytes.offset, bytes.length, StandardCharsets.UTF_8);
assertTrue(binaryFldStoredTest.equals(binaryValStored));
- /** fetch the string field and compare it's content with the original one */
+ /** fetch the string field and compare its content with the original one */
String stringFldStoredTest = docFromReader.getString("stringStored");
assertTrue(stringFldStoredTest.equals(binaryValStored));
@@ -87,7 +87,7 @@ public class TestBinaryDocument extends
Document docFromReader = reader.document(0);
assertTrue(docFromReader != null);
- /** fetch the binary compressed field and compare it's content with the original one */
+ /** fetch the binary compressed field and compare its content with the original one */
String binaryFldCompressedTest = new String(CompressionTools.decompress(docFromReader.getBinary("binaryCompressed")), StandardCharsets.UTF_8);
assertTrue(binaryFldCompressedTest.equals(binaryValCompressed));
assertTrue(CompressionTools.decompressString(docFromReader.getBinary("stringCompressed")).equals(binaryValCompressed));
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Sun Jan 4 14:53:12 2015
@@ -712,10 +712,10 @@ public class TestAddIndexes extends Luce
void close(boolean doWait) throws Throwable {
didClose = true;
if (doWait == false) {
- writer2.abortMerges();
+ writer2.rollback();
+ } else {
+ writer2.close();
}
- //writer2.close();
- writer2.rollback();
}
void closeDir() throws Throwable {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java Sun Jan 4 14:53:12 2015
@@ -37,7 +37,7 @@ import org.apache.lucene.util.TestUtil;
* Simple test that adds numeric terms, where each term has the
* totalTermFreq of its integer value, and checks that the totalTermFreq is correct.
*/
-// TODO: somehow factor this with BagOfPostings? its almost the same
+// TODO: somehow factor this with BagOfPostings? it's almost the same
@SuppressCodecs({"Direct", "Memory"}) // at night this makes like 200k/300k docs and will make Direct's heart beat!
public class TestBagOfPositions extends LuceneTestCase {
public void test() throws Exception {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Sun Jan 4 14:53:12 2015
@@ -395,6 +395,22 @@ public class TestConcurrentMergeSchedule
d.close();
}
+ public void testInvalidMaxMergeCountAndThreads() throws Exception {
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ try {
+ cms.setMaxMergesAndThreads(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, 3);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // good
+ }
+ try {
+ cms.setMaxMergesAndThreads(3, ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // good
+ }
+ }
+
public void testLiveMaxMergeCount() throws Exception {
Directory d = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -430,6 +446,9 @@ public class TestConcurrentMergeSchedule
}
};
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxMergeCount());
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxThreadCount());
+
cms.setMaxMergesAndThreads(5, 3);
iwc.setMergeScheduler(cms);
@@ -551,4 +570,75 @@ public class TestConcurrentMergeSchedule
w.rollback();
dir.close();
}
+
+ public void testDynamicDefaults() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxMergeCount());
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxThreadCount());
+ iwc.setMergeScheduler(cms);
+ iwc.setMaxBufferedDocs(2);
+ LogMergePolicy lmp = newLogMergePolicy();
+ lmp.setMergeFactor(2);
+ iwc.setMergePolicy(lmp);
+
+ IndexWriter w = new IndexWriter(dir, iwc);
+ w.addDocument(w.newDocument());
+ w.addDocument(w.newDocument());
+ // flush
+
+ w.addDocument(w.newDocument());
+ w.addDocument(w.newDocument());
+ // flush + merge
+
+ // CMS should have now set true values:
+ assertTrue(cms.getMaxMergeCount() != ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
+ assertTrue(cms.getMaxThreadCount() != ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
+ w.close();
+ dir.close();
+ }
+
+ public void testResetToAutoDefault() throws Exception {
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxMergeCount());
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxThreadCount());
+ cms.setMaxMergesAndThreads(4, 3);
+ assertEquals(4, cms.getMaxMergeCount());
+ assertEquals(3, cms.getMaxThreadCount());
+
+ try {
+ cms.setMaxMergesAndThreads(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, 4);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+
+ try {
+ cms.setMaxMergesAndThreads(4, ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+
+ cms.setMaxMergesAndThreads(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS);
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxMergeCount());
+ assertEquals(ConcurrentMergeScheduler.AUTO_DETECT_MERGES_AND_THREADS, cms.getMaxThreadCount());
+ }
+
+ public void testSpinningDefaults() throws Exception {
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ cms.setDefaultMaxMergesAndThreads(true);
+ assertEquals(1, cms.getMaxThreadCount());
+ assertEquals(2, cms.getMaxMergeCount());
+ }
+
+ public void testNonSpinningDefaults() throws Exception {
+ ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+ cms.setDefaultMaxMergesAndThreads(false);
+ int threadCount = cms.getMaxThreadCount();
+ assertTrue(threadCount >= 1);
+ assertTrue(threadCount <= 3);
+ assertEquals(cms.getMaxMergeCount(), 2+threadCount);
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Sun Jan 4 14:53:12 2015
@@ -915,7 +915,7 @@ public class TestDirectoryReader extends
reader.close();
- // Close the top reader, its the only one that should be closed
+ // Close the top reader, it's the only one that should be closed
assertEquals(1, closeCount[0]);
writer.close();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sun Jan 4 14:53:12 2015
@@ -48,7 +48,7 @@ public class TestDuelingCodecs extends L
public void setUp() throws Exception {
super.setUp();
- // for now its SimpleText vs Default(random postings format)
+ // for now it's SimpleText vs Default(random postings format)
// as this gives the best overall coverage. when we have more
// codecs we should probably pick 2 from Codec.availableCodecs()
@@ -123,7 +123,7 @@ public class TestDuelingCodecs extends L
*/
public static void createRandomIndex(int numdocs, RandomIndexWriter writer, long seed) throws IOException {
Random random = new Random(seed);
- // primary source for our data is from linefiledocs, its realistic.
+ // primary source for our data is from linefiledocs, it's realistic.
LineFileDocs lineFileDocs = new LineFileDocs(writer.w, random);
FieldTypes fieldTypes = writer.getFieldTypes();
fieldTypes.setMultiValued("sortedset");
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java Sun Jan 4 14:53:12 2015
@@ -191,4 +191,17 @@ public class TestFilterLeafReader extend
checkOverrideMethods(FilterLeafReader.FilterDocsAndPositionsEnum.class);
}
+ public void testUnwrap() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ w.addDocument(w.newDocument());
+ DirectoryReader dr = w.getReader();
+ LeafReader r = dr.leaves().get(0).reader();
+ FilterLeafReader r2 = new FilterLeafReader(r);
+ assertEquals(r, r2.getDelegate());
+ assertEquals(r, FilterLeafReader.unwrap(r2));
+ w.close();
+ dr.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java Sun Jan 4 14:53:12 2015
@@ -30,7 +30,7 @@ import org.apache.lucene.util.TestUtil;
public class TestForTooMuchCloning extends LuceneTestCase {
// Make sure we don't clone IndexInputs too frequently
- // during merging:
+ // during merging and searching:
public void test() throws Exception {
final MockDirectoryWrapper dir = newMockDirectory();
final TieredMergePolicy tmp = new TieredMergePolicy();
@@ -53,12 +53,14 @@ public class TestForTooMuchCloning exten
final IndexReader r = w.getReader();
w.close();
- final int cloneCount = dir.getInputCloneCount();
//System.out.println("merge clone count=" + cloneCount);
- assertTrue("too many calls to IndexInput.clone during merging: " + dir.getInputCloneCount(), cloneCount < 500);
+ assertTrue("too many calls to IndexInput.clone during merging: " + dir.getInputCloneCount(), dir.getInputCloneCount() < 500);
final IndexSearcher s = newSearcher(r);
-
+ // important: set this after newSearcher, it might have run checkindex
+ final int cloneCount = dir.getInputCloneCount();
+ // dir.setVerboseClone(true);
+
// MTQ that matches all terms so the AUTO_REWRITE should
// cutover to filter rewrite and reuse a single DocsEnum
// across all terms;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Jan 4 14:53:12 2015
@@ -2165,7 +2165,7 @@ public class TestIndexWriter extends Luc
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
final SetOnce<IndexWriter> iwRef = new SetOnce<>();
- iwc.setInfoStream(new RandomIndexWriter.TestPointInfoStream(iwc.getInfoStream(), new RandomIndexWriter.TestPoint() {
+ IndexWriter evilWriter = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
public void apply(String message) {
if ("startCommitMerge".equals(message)) {
@@ -2174,8 +2174,7 @@ public class TestIndexWriter extends Luc
iwRef.get().setKeepFullyDeletedSegments(true);
}
}
- }));
- IndexWriter evilWriter = new IndexWriter(dir, iwc);
+ });
iwRef.set(evilWriter);
for (int i = 0; i < 1000; i++) {
addDoc(evilWriter);
@@ -2362,10 +2361,12 @@ public class TestIndexWriter extends Luc
final CountDownLatch startCommit = new CountDownLatch(1);
final CountDownLatch finishCommit = new CountDownLatch(1);
- // infostream that "takes a long time" to commit
- InfoStream slowCommittingInfoStream = new InfoStream() {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(null);
+ // use an infostream that "takes a long time" to commit
+ final IndexWriter iw = RandomIndexWriter.mockIndexWriter(dir, iwc, new RandomIndexWriter.TestPoint() {
@Override
- public void message(String component, String message) {
+ public void apply(String message) {
if (message.equals("finishStartCommit")) {
startCommit.countDown();
try {
@@ -2375,20 +2376,7 @@ public class TestIndexWriter extends Luc
}
}
}
-
- @Override
- public boolean isEnabled(String component) {
- return true;
- }
-
- @Override
- public void close() throws IOException {}
- };
-
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(null);
- iwc.setInfoStream(slowCommittingInfoStream);
- final IndexWriter iw = new IndexWriter(dir, iwc);
+ });
new Thread() {
@Override
public void run() {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Sun Jan 4 14:53:12 2015
@@ -234,7 +234,7 @@ public class TestIndexWriterConfig exten
}
// Test Similarity:
- // we shouldnt assert what the default is, just that its not null.
+ // we shouldnt assert what the default is, just that it's not null.
assertTrue(IndexSearcher.getDefaultSimilarity() == conf.getSimilarity());
conf.setSimilarity(new MySimilarity());
assertEquals(MySimilarity.class, conf.getSimilarity().getClass());
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sun Jan 4 14:53:12 2015
@@ -2111,6 +2111,8 @@ public class TestIndexWriterExceptions e
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setInfoStream(evilInfoStream);
IndexWriter iw = new IndexWriter(dir, iwc);
+ // TODO: cutover to RandomIndexWriter.mockIndexWriter?
+ iw.enableTestPoints = true;
Document doc = iw.newDocument();
for (int i = 0; i < 10; i++) {
iw.addDocument(doc);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Sun Jan 4 14:53:12 2015
@@ -40,7 +40,7 @@ import org.apache.lucene.util.TestUtil;
import com.carrotsearch.randomizedtesting.SeedUtils;
/**
* Runs TestNRTThreads in a separate process, crashes the JRE in the middle
- * of execution, then runs checkindex to make sure its not corrupt.
+ * of execution, then runs checkindex to make sure it's not corrupt.
*/
public class TestIndexWriterOnJRECrash extends TestNRTThreads {
private Path tempDir;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java Sun Jan 4 14:53:12 2015
@@ -142,7 +142,7 @@ public class TestIndexWriterOutOfFileDes
dirCopy = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors.copy"));
Set<String> files = new HashSet<>();
for (String file : dir.listAll()) {
- dir.copy(dirCopy, file, file, IOContext.DEFAULT);
+ dirCopy.copyFrom(dir, file, file, IOContext.DEFAULT);
files.add(file);
}
dirCopy.sync(files);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Sun Jan 4 14:53:12 2015
@@ -434,11 +434,10 @@ public class TestIndexWriterReader exten
void close(boolean doWait) throws Throwable {
didClose = true;
if (doWait) {
- mainWriter.waitForMerges();
+ mainWriter.close();
} else {
- mainWriter.abortMerges();
+ mainWriter.rollback();
}
- mainWriter.close();
}
void closeDir() throws Throwable {
Copied: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java (from r1649173, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java?p2=lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java&r1=1649173&r2=1649347&rev=1649347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestInfoStream.java Sun Jan 4 14:53:12 2015
@@ -48,7 +48,7 @@ public class TestInfoStream extends Luce
}
});
IndexWriter iw = new IndexWriter(dir, iwc);
- iw.addDocument(new Document());
+ iw.addDocument(iw.newDocument());
iw.close();
dir.close();
}
@@ -76,7 +76,7 @@ public class TestInfoStream extends Luce
});
IndexWriter iw = new IndexWriter(dir, iwc);
iw.enableTestPoints = true;
- iw.addDocument(new Document());
+ iw.addDocument(iw.newDocument());
iw.close();
dir.close();
assertTrue(seenTestPoint.get());
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Sun Jan 4 14:53:12 2015
@@ -44,16 +44,6 @@ public class TestStressIndexing2 extends
static int maxBufferedDocs=3;
static int seed=0;
- public final class YieldTestPoint implements RandomIndexWriter.TestPoint {
-
- @Override
- public void apply(String name) {
- // if (name.equals("startCommit")) {
- if (random().nextInt(4) == 2)
- Thread.yield();
- }
- }
-//
public void testRandomIWReader() throws Throwable {
Directory dir = newDirectory();
@@ -147,7 +137,7 @@ public class TestStressIndexing2 extends
.setOpenMode(OpenMode.CREATE)
.setRAMBufferSizeMB(0.1)
.setMaxBufferedDocs(maxBufferedDocs)
- .setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
+ .setMergePolicy(newLogMergePolicy()), random());
w.commit();
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
@@ -202,7 +192,7 @@ public class TestStressIndexing2 extends
.setMaxBufferedDocs(maxBufferedDocs)
.setIndexerThreadPool(new DocumentsWriterPerThreadPool(maxThreadStates))
.setReaderPooling(doReaderPooling)
- .setMergePolicy(newLogMergePolicy()), new YieldTestPoint());
+ .setMergePolicy(newLogMergePolicy()), random());
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setNoCFSRatio(0.0);
lmp.setMergeFactor(mergeFactor);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Sun Jan 4 14:53:12 2015
@@ -180,11 +180,9 @@ public class TestTermVectorsWriter exten
fieldTypes.enableTermVectorPositions("field");
fieldTypes.setMultiValued("field");
Document doc = w.newDocument();
- try (TokenStream stream = analyzer.tokenStream("field", "abcd ")) {
- stream.reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
- TokenStream cachedStream = new CachingTokenFilter(stream);
- doc.addLargeText("field", cachedStream);
- doc.addLargeText("field", cachedStream);
+ try (TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", "abcd "))) {
+ doc.addLargeText("field", stream);
+ doc.addLargeText("field", stream);
w.addDocument(doc);
}
w.close();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java Sun Jan 4 14:53:12 2015
@@ -18,19 +18,29 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageTester;
+
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
public class TestLRUFilterCache extends LuceneTestCase {
@@ -38,7 +48,7 @@ public class TestLRUFilterCache extends
private static final FilterCachingPolicy NEVER_CACHE = new FilterCachingPolicy() {
@Override
- public void onCache(Filter filter) {}
+ public void onUse(Filter filter) {}
@Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@@ -47,6 +57,15 @@ public class TestLRUFilterCache extends
};
+ public void testFilterRamBytesUsed() {
+ final Filter simpleFilter = new QueryWrapperFilter(new TermQuery(new Term("some_field", "some_term")));
+ final long actualRamBytesUsed = RamUsageTester.sizeOf(simpleFilter);
+ final long ramBytesUsed = LRUFilterCache.FILTER_DEFAULT_RAM_BYTES_USED;
+ // we cannot assert exactly that the constant is correct since actual
+ // memory usage depends on JVM implementations and settings (eg. UseCompressedOops)
+ assertEquals(actualRamBytesUsed, ramBytesUsed, actualRamBytesUsed / 2);
+ }
+
public void testConcurrency() throws Throwable {
final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(20), 1 + random().nextInt(10000));
Directory dir = newDirectory();
@@ -149,36 +168,368 @@ public class TestLRUFilterCache extends
final Filter red = new QueryWrapperFilter(new TermQuery(new Term("color", "red")));
final Filter green = new QueryWrapperFilter(new TermQuery(new Term("color", "green")));
- assertEquals(Collections.emptySet(), filterCache.cachedFilters());
+ assertEquals(Collections.emptyList(), filterCache.cachedFilters());
// the filter is not cached on any segment: no changes
searcher.search(new ConstantScoreQuery(filterCache.doCache(green, NEVER_CACHE)), 1);
- assertEquals(Collections.emptySet(), filterCache.cachedFilters());
+ assertEquals(Collections.emptyList(), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(red, FilterCachingPolicy.ALWAYS_CACHE)), 1);
- assertEquals(Collections.singleton(red), filterCache.cachedFilters());
+ assertEquals(Collections.singletonList(red), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(green, FilterCachingPolicy.ALWAYS_CACHE)), 1);
- assertEquals(new HashSet<>(Arrays.asList(red, green)), filterCache.cachedFilters());
+ assertEquals(Arrays.asList(red, green), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(red, FilterCachingPolicy.ALWAYS_CACHE)), 1);
- assertEquals(new HashSet<>(Arrays.asList(red, green)), filterCache.cachedFilters());
+ assertEquals(Arrays.asList(green, red), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(blue, FilterCachingPolicy.ALWAYS_CACHE)), 1);
- assertEquals(new HashSet<>(Arrays.asList(red, blue)), filterCache.cachedFilters());
+ assertEquals(Arrays.asList(red, blue), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(blue, FilterCachingPolicy.ALWAYS_CACHE)), 1);
- assertEquals(new HashSet<>(Arrays.asList(red, blue)), filterCache.cachedFilters());
+ assertEquals(Arrays.asList(red, blue), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(green, FilterCachingPolicy.ALWAYS_CACHE)), 1);
- assertEquals(new HashSet<>(Arrays.asList(green, blue)), filterCache.cachedFilters());
+ assertEquals(Arrays.asList(blue, green), filterCache.cachedFilters());
searcher.search(new ConstantScoreQuery(filterCache.doCache(red, NEVER_CACHE)), 1);
- assertEquals(new HashSet<>(Arrays.asList(green, blue)), filterCache.cachedFilters());
+ assertEquals(Arrays.asList(blue, green), filterCache.cachedFilters());
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testCache() throws IOException {
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+ final int numDocs = atLeast(10);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = w.newDocument();
+ doc.addAtom("color", RandomPicks.randomFrom(random(), Arrays.asList("blue", "red", "green")));
+ w.addDocument(doc);
+ }
+ final DirectoryReader reader = w.getReader();
+ final LeafReaderContext leaf1 = reader.leaves().get(0);
+
+ Filter filter1 = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
+ // different instance yet equal
+ Filter filter2 = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
+
+ final LRUFilterCache filterCache = new LRUFilterCache(Integer.MAX_VALUE, Long.MAX_VALUE);
+ final Filter cachedFilter1 = filterCache.doCache(filter1, FilterCachingPolicy.ALWAYS_CACHE);
+ DocIdSet cached1 = cachedFilter1.getDocIdSet(leaf1, null);
+
+ final Filter cachedFilter2 = filterCache.doCache(filter2, NEVER_CACHE);
+ DocIdSet cached2 = cachedFilter2.getDocIdSet(leaf1, null);
+ assertSame(cached1, cached2);
+
+ filterCache.assertConsistent();
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testClearFilter() throws IOException {
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+ final int numDocs = atLeast(10);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = w.newDocument();
+ doc.addAtom("color", random().nextBoolean() ? "red" : "blue");
+ w.addDocument(doc);
+ }
+ final DirectoryReader reader = w.getReader();
+ final LeafReaderContext leaf1 = reader.leaves().get(0);
+
+ final Filter filter1 = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
+ // different instance yet equal
+ final Filter filter2 = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
+
+ final LRUFilterCache filterCache = new LRUFilterCache(Integer.MAX_VALUE, Long.MAX_VALUE);
+
+ final Filter cachedFilter1 = filterCache.doCache(filter1, FilterCachingPolicy.ALWAYS_CACHE);
+ cachedFilter1.getDocIdSet(leaf1, null);
+
+ filterCache.clearFilter(filter2);
+
+ assertTrue(filterCache.cachedFilters().isEmpty());
+ filterCache.assertConsistent();
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ // This test makes sure that by making the same assumptions as LRUFilterCache, RAMUsageTester
+ // computes the same memory usage.
+ public void testRamBytesUsedAgreesWithRamUsageTester() throws IOException {
+ final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(5), 1 + random().nextInt(10000));
+ // an accumulator that only sums up memory usage of referenced filters and doc id sets
+ final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() {
+ @Override
+ public long accumulateObject(Object o, long shallowSize, Map<Field,Object> fieldValues, Collection<Object> queue) {
+ if (o instanceof DocIdSet) {
+ return ((DocIdSet) o).ramBytesUsed();
+ }
+ if (o instanceof Filter) {
+ return filterCache.ramBytesUsed((Filter) o);
+ }
+ if (o.getClass().getSimpleName().equals("SegmentCoreReaders")) {
+ // do not take core cache keys into account
+ return 0;
+ }
+ if (o instanceof Map) {
+ Map<?,?> map = (Map<?,?>) o;
+ queue.addAll(map.keySet());
+ queue.addAll(map.values());
+ final long sizePerEntry = o instanceof LinkedHashMap
+ ? LRUFilterCache.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY
+ : LRUFilterCache.HASHTABLE_RAM_BYTES_PER_ENTRY;
+ return sizePerEntry * map.size();
+ }
+ // follow links to other objects, but ignore their memory usage
+ super.accumulateObject(o, shallowSize, fieldValues, queue);
+ return 0;
+ }
+ @Override
+ public long accumulateArray(Object array, long shallowSize, List<Object> values, Collection<Object> queue) {
+ // follow links to other objects, but ignore their memory usage
+ super.accumulateArray(array, shallowSize, values, queue);
+ return 0;
+ }
+ };
+
+ Directory dir = newDirectory();
+ // serial merges so that segments do not get closed while we are measuring ram usage
+ // with RamUsageTester
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler());
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ final List<String> colors = Arrays.asList("blue", "red", "green", "yellow");
+
+ final int iters = atLeast(5);
+ for (int iter = 0; iter < iters; ++iter) {
+ final int numDocs = atLeast(10);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = w.newDocument();
+ doc.addAtom("color", RandomPicks.randomFrom(random(), colors));
+ w.addDocument(doc);
+ }
+ try (final DirectoryReader reader = w.getReader()) {
+ final IndexSearcher searcher = new IndexSearcher(reader);
+ for (int i = 0; i < 3; ++i) {
+ final Filter filter = new QueryWrapperFilter(new TermQuery(new Term("color", RandomPicks.randomFrom(random(), colors))));
+ searcher.search(new ConstantScoreQuery(filterCache.doCache(filter, MAYBE_CACHE_POLICY)), 1);
+ }
+ }
+ filterCache.assertConsistent();
+ assertEquals(RamUsageTester.sizeOf(filterCache, acc), filterCache.ramBytesUsed());
+ }
+
+ w.close();
+ dir.close();
+ }
+
+ /** A filter that produces empty sets. */
+ private static class DummyFilter extends Filter {
+
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ return null;
+ }
+
+ }
+
+ // Test what happens when the cache contains only filters and doc id sets
+ // that require very little memory. In that case most of the memory is taken
+ // by the cache itself, not cache entries, and we want to make sure that
+ // memory usage is not grossly underestimated.
+ public void testRamBytesUsedConstantEntryOverhead() throws IOException {
+ final LRUFilterCache filterCache = new LRUFilterCache(1000000, 10000000);
+
+ final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() {
+ @Override
+ public long accumulateObject(Object o, long shallowSize, Map<Field,Object> fieldValues, Collection<Object> queue) {
+ if (o instanceof DocIdSet) {
+ return ((DocIdSet) o).ramBytesUsed();
+ }
+ if (o instanceof Filter) {
+ return filterCache.ramBytesUsed((Filter) o);
+ }
+ if (o.getClass().getSimpleName().equals("SegmentCoreReaders")) {
+ // do not follow references to core cache keys
+ return 0;
+ }
+ return super.accumulateObject(o, shallowSize, fieldValues, queue);
+ }
+ };
+
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ w.addDocument(w.newDocument());
+ }
+ final DirectoryReader reader = w.getReader();
+ final IndexSearcher searcher = new IndexSearcher(reader);
+
+ final int numFilters = atLeast(1000);
+ for (int i = 0; i < numFilters; ++i) {
+ final Filter filter = new DummyFilter();
+ final Filter cached = filterCache.doCache(filter, FilterCachingPolicy.ALWAYS_CACHE);
+ searcher.search(new ConstantScoreQuery(cached), 1);
+ }
+
+ final long actualRamBytesUsed = RamUsageTester.sizeOf(filterCache, acc);
+ final long expectedRamBytesUsed = filterCache.ramBytesUsed();
+ // error < 30%
+ assertEquals(actualRamBytesUsed, expectedRamBytesUsed, 30 * actualRamBytesUsed / 100);
reader.close();
w.close();
dir.close();
}
+ public void testOnUse() throws IOException {
+ final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(5), 1 + random().nextInt(1000));
+
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+ final int numDocs = atLeast(10);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = w.newDocument();
+ doc.addAtom("color", RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green", "yellow")));
+ w.addDocument(doc);
+ if (random().nextBoolean()) {
+ w.getReader().close();
+ }
+ }
+ final DirectoryReader reader = w.getReader();
+ final IndexSearcher searcher = new IndexSearcher(reader);
+
+ final Map<Filter, Integer> actualCounts = new HashMap<>();
+ final Map<Filter, Integer> expectedCounts = new HashMap<>();
+
+ final FilterCachingPolicy countingPolicy = new FilterCachingPolicy() {
+
+ @Override
+ public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
+ return random().nextBoolean();
+ }
+
+ @Override
+ public void onUse(Filter filter) {
+ expectedCounts.put(filter, 1 + expectedCounts.getOrDefault(filter, 0));
+ }
+ };
+
+ Filter[] filters = new Filter[10 + random().nextInt(10)];
+ Filter[] cachedFilters = new Filter[filters.length];
+ for (int i = 0; i < filters.length; ++i) {
+ filters[i] = new QueryWrapperFilter(new TermQuery(new Term("color", RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green", "yellow")))));
+ cachedFilters[i] = filterCache.doCache(filters[i], countingPolicy);
+ }
+
+ for (int i = 0; i < 20; ++i) {
+ final int idx = random().nextInt(filters.length);
+ searcher.search(new ConstantScoreQuery(cachedFilters[idx]), 1);
+ actualCounts.put(filters[idx], 1 + actualCounts.getOrDefault(filters[idx], 0));
+ }
+
+ assertEquals(actualCounts, expectedCounts);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testStats() throws IOException {
+ final LRUFilterCache filterCache = new LRUFilterCache(1, 10000000);
+
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+ final List<String> colors = Arrays.asList("blue", "red", "green", "yellow");
+
+ for (int i = 0; i < 10; ++i) {
+ Document doc = w.newDocument();
+ doc.addAtom("color", RandomPicks.randomFrom(random(), colors));
+ w.addDocument(doc);
+ if (random().nextBoolean()) {
+ w.getReader().close();
+ }
+ }
+
+ final DirectoryReader reader = w.getReader();
+ final int segmentCount = reader.leaves().size();
+ final IndexSearcher searcher = new IndexSearcher(reader);
+ final Filter filter = new QueryWrapperFilter(new TermQuery(new Term("color", "red")));
+ final Filter filter2 = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
+
+ // first pass, lookups without caching that all miss
+ Filter cached = filterCache.doCache(filter, NEVER_CACHE);
+ for (int i = 0; i < 10; ++i) {
+ searcher.search(new ConstantScoreQuery(cached), 1);
+ }
+ assertEquals(10 * segmentCount, filterCache.getTotalCount());
+ assertEquals(0, filterCache.getHitCount());
+ assertEquals(10 * segmentCount, filterCache.getMissCount());
+ assertEquals(0, filterCache.getCacheCount());
+ assertEquals(0, filterCache.getEvictionCount());
+ assertEquals(0, filterCache.getCacheSize());
+
+ // second pass, lookups + caching, only the first one is a miss
+ cached = filterCache.doCache(filter, FilterCachingPolicy.ALWAYS_CACHE);
+ for (int i = 0; i < 10; ++i) {
+ searcher.search(new ConstantScoreQuery(cached), 1);
+ }
+ assertEquals(20 * segmentCount, filterCache.getTotalCount());
+ assertEquals(9 * segmentCount, filterCache.getHitCount());
+ assertEquals(11 * segmentCount, filterCache.getMissCount());
+ assertEquals(1 * segmentCount, filterCache.getCacheCount());
+ assertEquals(0, filterCache.getEvictionCount());
+ assertEquals(1 * segmentCount, filterCache.getCacheSize());
+
+ // third pass lookups without caching, we only have hits
+ cached = filterCache.doCache(filter, NEVER_CACHE);
+ for (int i = 0; i < 10; ++i) {
+ searcher.search(new ConstantScoreQuery(cached), 1);
+ }
+ assertEquals(30 * segmentCount, filterCache.getTotalCount());
+ assertEquals(19 * segmentCount, filterCache.getHitCount());
+ assertEquals(11 * segmentCount, filterCache.getMissCount());
+ assertEquals(1 * segmentCount, filterCache.getCacheCount());
+ assertEquals(0, filterCache.getEvictionCount());
+ assertEquals(1 * segmentCount, filterCache.getCacheSize());
+
+ // fourth pass with a different filter which will trigger evictions since the size is 1
+ cached = filterCache.doCache(filter2, FilterCachingPolicy.ALWAYS_CACHE);
+ for (int i = 0; i < 10; ++i) {
+ searcher.search(new ConstantScoreQuery(cached), 1);
+ }
+ assertEquals(40 * segmentCount, filterCache.getTotalCount());
+ assertEquals(28 * segmentCount, filterCache.getHitCount());
+ assertEquals(12 * segmentCount, filterCache.getMissCount());
+ assertEquals(2 * segmentCount, filterCache.getCacheCount());
+ assertEquals(1 * segmentCount, filterCache.getEvictionCount());
+ assertEquals(1 * segmentCount, filterCache.getCacheSize());
+
+ // now close, causing evictions due to the closing of segment cores
+ reader.close();
+ w.close();
+ assertEquals(40 * segmentCount, filterCache.getTotalCount());
+ assertEquals(28 * segmentCount, filterCache.getHitCount());
+ assertEquals(12 * segmentCount, filterCache.getMissCount());
+ assertEquals(2 * segmentCount, filterCache.getCacheCount());
+ assertEquals(2 * segmentCount, filterCache.getEvictionCount());
+ assertEquals(0, filterCache.getCacheSize());
+
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestRegexpQuery.java Sun Jan 4 14:53:12 2015
@@ -116,7 +116,7 @@ public class TestRegexpQuery extends Luc
/**
* Test a corner case for backtracking: In this case the term dictionary has
- * 493432 followed by 49344. When backtracking from 49343... to 4934, its
+ * 493432 followed by 49344. When backtracking from 49343... to 4934, it's
* necessary to test that 4934 itself is ok before trying to append more
* characters.
*/
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java Sun Jan 4 14:53:12 2015
@@ -145,7 +145,7 @@ public class TestSearcherManager extends
@Override
protected IndexSearcher getCurrentSearcher() throws Exception {
if (random().nextInt(10) == 7) {
- // NOTE: not best practice to call maybeReopen
+ // NOTE: not best practice to call maybeRefresh
// synchronous to your search threads, but still we
// test as apps will presumably do this for
// simplicity:
@@ -250,7 +250,7 @@ public class TestSearcherManager extends
try {
triedReopen.set(true);
if (VERBOSE) {
- System.out.println("NOW call maybeReopen");
+ System.out.println("NOW call maybeRefresh");
}
searcherManager.maybeRefresh();
success.set(true);
@@ -461,7 +461,9 @@ public class TestSearcherManager extends
new FilterDirectoryReader.SubReaderWrapper() {
@Override
public LeafReader wrap(LeafReader reader) {
- return new MyFilterLeafReader(reader);
+ FilterLeafReader wrapped = new MyFilterLeafReader(reader);
+ assertEquals(reader, wrapped.getDelegate());
+ return wrapped;
}
});
}
@@ -476,7 +478,12 @@ public class TestSearcherManager extends
public void testCustomDirectoryReader() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- DirectoryReader reader = new MyFilterDirectoryReader(w.getReader());
+ DirectoryReader nrtReader = w.getReader();
+
+ FilterDirectoryReader reader = new MyFilterDirectoryReader(nrtReader);
+ assertEquals(nrtReader, reader.getDelegate());
+ assertEquals(nrtReader, FilterDirectoryReader.unwrap(reader));
+
SearcherManager mgr = new SearcherManager(reader, null);
for(int i=0;i<10;i++) {
w.addDocument(w.newDocument());
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java Sun Jan 4 14:53:12 2015
@@ -171,7 +171,7 @@ public class TestSubScorerFreqs extends
boolean includeOptional = occur.contains("SHOULD");
for (int i = 0; i < maxDocs; i++) {
Map<Query, Float> doc0 = c.docCounts.get(i);
- // Y doesnt exist in the index, so its not in the scorer tree
+ // Y doesnt exist in the index, so it's not in the scorer tree
assertEquals(4, doc0.size());
assertEquals(1.0F, doc0.get(aQuery), FLOAT_TOLERANCE);
assertEquals(4.0F, doc0.get(dQuery), FLOAT_TOLERANCE);
@@ -180,7 +180,7 @@ public class TestSubScorerFreqs extends
}
Map<Query, Float> doc1 = c.docCounts.get(++i);
- // Y doesnt exist in the index, so its not in the scorer tree
+ // Y doesnt exist in the index, so it's not in the scorer tree
assertEquals(4, doc1.size());
assertEquals(1.0F, doc1.get(aQuery), FLOAT_TOLERANCE);
assertEquals(1.0F, doc1.get(dQuery), FLOAT_TOLERANCE);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java Sun Jan 4 14:53:12 2015
@@ -69,7 +69,7 @@ public class TestSimilarity2 extends Luc
sims.add(new LMJelinekMercerSimilarity(0.7f));
}
- /** because of stupid things like querynorm, its possible we computeStats on a field that doesnt exist at all
+ /** because of stupid things like querynorm, it's possible we computeStats on a field that doesnt exist at all
* test this against a totally empty index, to make sure sims handle it
*/
public void testEmptyIndex() throws Exception {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java Sun Jan 4 14:53:12 2015
@@ -17,21 +17,27 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.IOException;
import java.lang.reflect.Method;
+import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
-public class TestFilterDirectory extends LuceneTestCase {
+public class TestFilterDirectory extends BaseDirectoryTestCase {
+ @Override
+ protected Directory getDirectory(Path path) throws IOException {
+ return new FilterDirectory(new RAMDirectory());
+ }
+
@Test
public void testOverrides() throws Exception {
// verify that all methods of Directory are overridden by FilterDirectory,
// except those under the 'exclude' list
Set<Method> exclude = new HashSet<>();
- exclude.add(Directory.class.getMethod("copy", Directory.class, String.class, String.class, IOContext.class));
+ exclude.add(Directory.class.getMethod("copyFrom", Directory.class, String.class, String.class, IOContext.class));
exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class));
for (Method m : FilterDirectory.class.getMethods()) {
if (m.getDeclaringClass() == Directory.class) {
@@ -39,5 +45,12 @@ public class TestFilterDirectory extends
}
}
}
-
+
+ public void testUnwrap() throws IOException {
+ Directory dir = FSDirectory.open(createTempDir());
+ FilterDirectory dir2 = new FilterDirectory(dir);
+ assertEquals(dir, dir2.getDelegate());
+ assertEquals(dir, FilterDirectory.unwrap(dir2));
+ dir2.close();
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java Sun Jan 4 14:53:12 2015
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
public class TestMockDirectoryWrapper extends LuceneTestCase {
@@ -33,9 +34,9 @@ public class TestMockDirectoryWrapper ex
fail();
} catch (Exception expected) {
assertTrue(expected.getMessage().contains("there are still open locks"));
+ } finally {
+ IOUtils.closeWhileHandlingException(iw);
}
- iw.close();
- dir.close();
}
public void testFailIfIndexWriterNotClosedChangeLockFactory() throws IOException {
@@ -46,9 +47,9 @@ public class TestMockDirectoryWrapper ex
fail();
} catch (Exception expected) {
assertTrue(expected.getMessage().contains("there are still open locks"));
+ } finally {
+ IOUtils.closeWhileHandlingException(iw);
}
- iw.close();
- dir.close();
}
public void testDiskFull() throws IOException {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java Sun Jan 4 14:53:12 2015
@@ -39,7 +39,7 @@ import org.apache.lucene.util.TestUtil;
public class TestNRTCachingDirectory extends BaseDirectoryTestCase {
- // TODO: RAMDir used here, because its still too slow to use e.g. SimpleFS
+ // TODO: RAMDir used here, because it's still too slow to use e.g. SimpleFS
// for the threads tests... maybe because of the synchronization in listAll?
// would be good to investigate further...
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java Sun Jan 4 14:53:12 2015
@@ -24,15 +24,19 @@ import java.util.Map;
public class TestFrequencyTrackingRingBuffer extends LuceneTestCase {
- private static <T> void assertBuffer(FrequencyTrackingRingBuffer<T> buffer, int maxSize, List<T> items) {
- final List<T> recentItems;
+ private static void assertBuffer(FrequencyTrackingRingBuffer buffer, int maxSize, int sentinel, List<Integer> items) {
+ final List<Integer> recentItems;
if (items.size() <= maxSize) {
- recentItems = items;
+ recentItems = new ArrayList<>();
+ for (int i = items.size(); i < maxSize; ++i) {
+ recentItems.add(sentinel);
+ }
+ recentItems.addAll(items);
} else {
recentItems = items.subList(items.size() - maxSize, items.size());
}
- final Map<T, Integer> expectedFrequencies = new HashMap<T, Integer>();
- for (T item : recentItems) {
+ final Map<Integer, Integer> expectedFrequencies = new HashMap<Integer, Integer>();
+ for (Integer item : recentItems) {
final Integer freq = expectedFrequencies.get(item);
if (freq == null) {
expectedFrequencies.put(item, 1);
@@ -46,18 +50,29 @@ public class TestFrequencyTrackingRingBu
public void test() {
final int iterations = atLeast(100);
for (int i = 0; i < iterations; ++i) {
- final int maxSize = 1 + random().nextInt(100);
- final int numitems = random().nextInt(500);
+ final int maxSize = 2 + random().nextInt(100);
+ final int numitems = random().nextInt(5000);
final int maxitem = 1 + random().nextInt(100);
List<Integer> items = new ArrayList<>();
- FrequencyTrackingRingBuffer<Integer> buffer = new FrequencyTrackingRingBuffer<>(maxSize);
+ final int sentinel = random().nextInt(200);
+ FrequencyTrackingRingBuffer buffer = new FrequencyTrackingRingBuffer(maxSize, sentinel);
for (int j = 0; j < numitems; ++j) {
final Integer item = random().nextInt(maxitem);
items.add(item);
buffer.add(item);
}
- assertBuffer(buffer, maxSize, items);
+ assertBuffer(buffer, maxSize, sentinel, items);
+ }
+ }
+
+ public void testRamBytesUsed() {
+ final int maxSize = 2 + random().nextInt(10000);
+ final int sentinel = random().nextInt();
+ FrequencyTrackingRingBuffer buffer = new FrequencyTrackingRingBuffer(maxSize, sentinel);
+ for (int i = 0; i < 10000; ++i) {
+ buffer.add(random().nextInt());
}
+ assertEquals(RamUsageTester.sizeOf(buffer), buffer.ramBytesUsed());
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java Sun Jan 4 14:53:12 2015
@@ -17,8 +17,27 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AccessMode;
+import java.nio.file.FileStore;
+import java.nio.file.FileSystem;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.OpenOption;
import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.lucene.mockfile.FilterFileStore;
+import org.apache.lucene.mockfile.FilterFileSystem;
+import org.apache.lucene.mockfile.FilterFileSystemProvider;
+import org.apache.lucene.mockfile.FilterPath;
/** Simple test methods for IOUtils */
public class TestIOUtils extends LuceneTestCase {
@@ -78,4 +97,284 @@ public class TestIOUtils extends LuceneT
// no exception
// actually deletes file2
}
+
+ public void testSpinsBasics() throws Exception {
+ Path dir = createTempDir();
+ // no exception, directory exists
+ IOUtils.spins(dir);
+ Path file = dir.resolve("exists");
+ Files.createFile(file);
+ // no exception, file exists
+ IOUtils.spins(file);
+
+ // exception: file doesn't exist
+ Path fake = dir.resolve("nonexistent");
+ try {
+ IOUtils.spins(fake);
+ fail();
+ } catch (IOException expected) {
+ // ok
+ }
+ }
+
+ // fake up a filestore to test some underlying methods
+ static class MockFileStore extends FilterFileStore {
+ final String description;
+ final String type;
+
+ MockFileStore(FileStore delegate, String description) {
+ this(delegate, description, "mockfs");
+ }
+
+ MockFileStore(FileStore delegate, String description, String type) {
+ super(delegate, "justafake://");
+ this.description = description;
+ this.type = type;
+ }
+
+ @Override
+ public String type() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return description;
+ }
+ }
+
+ public void testGetBlockDevice() throws Exception {
+ Path dir = createTempDir();
+ FileStore actual = Files.getFileStore(dir);
+
+ assertEquals("/dev/sda1", IOUtils.getBlockDevice(new MockFileStore(actual, "/ (/dev/sda1)")));
+ assertEquals("/dev/sda1", IOUtils.getBlockDevice(new MockFileStore(actual, "/test/ space(((trash)))/ (/dev/sda1)")));
+ assertEquals("notreal", IOUtils.getBlockDevice(new MockFileStore(actual, "/ (notreal)")));
+ }
+
+ public void testGetMountPoint() throws Exception {
+ Path dir = createTempDir();
+ FileStore actual = Files.getFileStore(dir);
+
+ assertEquals("/", IOUtils.getMountPoint(new MockFileStore(actual, "/ (/dev/sda1)")));
+ assertEquals("/test/ space(((trash)))/", IOUtils.getMountPoint(new MockFileStore(actual, "/test/ space(((trash)))/ (/dev/sda1)")));
+ assertEquals("/", IOUtils.getMountPoint(new MockFileStore(actual, "/ (notreal)")));
+ }
+
+ /** mock linux that takes mappings of test files, to their associated filesystems.
+ * it will chroot /dev and /sys requests to root, so you can mock those too.
+ * <p>
+ * It is hacky by definition, so don't try putting it around a complex chain or anything.
+ * Use FilterPath.unwrap
+ */
+ static class MockLinuxFileSystemProvider extends FilterFileSystemProvider {
+ final Map<String,FileStore> filesToStore;
+ final Path root;
+
+ public MockLinuxFileSystemProvider(FileSystem delegateInstance, final Map<String,FileStore> filesToStore, Path root) {
+ super("mocklinux://", delegateInstance);
+ final Collection<FileStore> allStores = new HashSet<>(filesToStore.values());
+ this.fileSystem = new FilterFileSystem(this, delegateInstance) {
+ @Override
+ public Iterable<FileStore> getFileStores() {
+ return allStores;
+ }
+
+ @Override
+ public Path getPath(String first, String... more) {
+ return new MockLinuxPath(super.getPath(first, more), this);
+ }
+ };
+ this.filesToStore = filesToStore;
+ this.root = root;
+ }
+
+ @Override
+ public FileStore getFileStore(Path path) throws IOException {
+ FileStore ret = filesToStore.get(path.toString());
+ if (ret == null) {
+ throw new IllegalArgumentException("this mock doesnt know wtf to do with: " + path);
+ }
+ // act like the linux fs provider here, return a crazy rootfs one
+ if (ret.toString().startsWith(root + " (")) {
+ return new MockFileStore(ret, root + " (rootfs)", "rootfs");
+ }
+
+ return ret;
+ }
+
+ Path maybeChroot(Path path) {
+ if (path.toAbsolutePath().startsWith("/sys") || path.toAbsolutePath().startsWith("/dev")) {
+ // map to our chrooted location;
+ return path.getRoot().resolve(root).resolve(path.toString().substring(1));
+ } else {
+ return path;
+ }
+ }
+
+ @Override
+ public void checkAccess(Path path, AccessMode... modes) throws IOException {
+ // TODO: kinda screwed up how we do this, but it's easy to get lost. just unravel completely.
+ delegate.checkAccess(FilterPath.unwrap(path), modes);
+ }
+
+ @Override
+ public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
+ return super.newInputStream(maybeChroot(path), options);
+ }
+
+ class MockLinuxPath extends FilterPath {
+ MockLinuxPath(Path delegate, FileSystem fileSystem) {
+ super(delegate, fileSystem);
+ }
+
+ @Override
+ public Path toRealPath(LinkOption... options) throws IOException {
+ Path p = maybeChroot(this);
+ if (p == this) {
+ return super.toRealPath(options);
+ } else {
+ return p.toRealPath(options);
+ }
+ }
+
+ @Override
+ protected Path wrap(Path other) {
+ return new MockLinuxPath(other, fileSystem);
+ }
+ }
+ }
+
+ public void testGetFileStore() throws Exception {
+ Path dir = createTempDir();
+ dir = FilterPath.unwrap(dir).toRealPath();
+
+ // now we can create some fake mount points:
+ FileStore root = new MockFileStore(Files.getFileStore(dir), dir.toString() + " (/dev/sda1)");
+ FileStore usr = new MockFileStore(Files.getFileStore(dir), dir.resolve("usr").toString() + " (/dev/sda2)");
+
+ // associate some preset files to these
+ Map<String,FileStore> mappings = new HashMap<>();
+ mappings.put(dir.toString(), root);
+ mappings.put(dir.resolve("foo.txt").toString(), root);
+ mappings.put(dir.resolve("usr").toString(), usr);
+ mappings.put(dir.resolve("usr/bar.txt").toString(), usr);
+
+ FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+ Path mockPath = mockLinux.getPath(dir.toString());
+
+ // sanity check our mock:
+ assertSame(usr, Files.getFileStore(mockPath.resolve("usr")));
+ assertSame(usr, Files.getFileStore(mockPath.resolve("usr/bar.txt")));
+ // for root filesystem we get a crappy one
+ assertNotSame(root, Files.getFileStore(mockPath));
+ assertNotSame(usr, Files.getFileStore(mockPath));
+ assertNotSame(root, Files.getFileStore(mockPath.resolve("foo.txt")));
+ assertNotSame(usr, Files.getFileStore(mockPath.resolve("foo.txt")));
+
+ // now test our method:
+ assertSame(usr, IOUtils.getFileStore(mockPath.resolve("usr")));
+ assertSame(usr, IOUtils.getFileStore(mockPath.resolve("usr/bar.txt")));
+ assertSame(root, IOUtils.getFileStore(mockPath));
+ assertSame(root, IOUtils.getFileStore(mockPath.resolve("foo.txt")));
+ }
+
+ public void testTmpfsDoesntSpin() throws Exception {
+ Path dir = createTempDir();
+ dir = FilterPath.unwrap(dir).toRealPath();
+
+ // fake tmpfs
+ FileStore root = new MockFileStore(Files.getFileStore(dir), dir.toString() + " (/dev/sda1)", "tmpfs");
+ Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+ FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+
+ Path mockPath = mockLinux.getPath(dir.toString());
+ assertFalse(IOUtils.spinsLinux(mockPath));
+ }
+
+ public void testNfsSpins() throws Exception {
+ Path dir = createTempDir();
+ dir = FilterPath.unwrap(dir).toRealPath();
+
+ // fake nfs
+ FileStore root = new MockFileStore(Files.getFileStore(dir), dir.toString() + " (somenfsserver:/some/mount)", "nfs");
+ Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+ FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+
+ Path mockPath = mockLinux.getPath(dir.toString());
+ assertTrue(IOUtils.spinsLinux(mockPath));
+ }
+
+ public void testSSD() throws Exception {
+ assumeFalse("windows is not supported", Constants.WINDOWS);
+ Path dir = createTempDir();
+ dir = FilterPath.unwrap(dir).toRealPath();
+
+ // fake ssd
+ FileStore root = new MockFileStore(Files.getFileStore(dir), dir.toString() + " (/dev/zzz1)");
+ // make a fake /dev/zzz1 for it
+ Path devdir = dir.resolve("dev");
+ Files.createDirectories(devdir);
+ Files.createFile(devdir.resolve("zzz1"));
+ // make a fake /sys/block/zzz/queue/rotational file for it
+ Path sysdir = dir.resolve("sys").resolve("block").resolve("zzz").resolve("queue");
+ Files.createDirectories(sysdir);
+ try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
+ o.write("0\n".getBytes(StandardCharsets.US_ASCII));
+ }
+ Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+ FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+
+ Path mockPath = mockLinux.getPath(dir.toString());
+ assertFalse(IOUtils.spinsLinux(mockPath));
+ }
+
+ public void testRotatingPlatters() throws Exception {
+ assumeFalse("windows is not supported", Constants.WINDOWS);
+ Path dir = createTempDir();
+ dir = FilterPath.unwrap(dir).toRealPath();
+
+ // fake ssd
+ FileStore root = new MockFileStore(Files.getFileStore(dir), dir.toString() + " (/dev/zzz1)");
+ // make a fake /dev/zzz1 for it
+ Path devdir = dir.resolve("dev");
+ Files.createDirectories(devdir);
+ Files.createFile(devdir.resolve("zzz1"));
+ // make a fake /sys/block/zzz/queue/rotational file for it
+ Path sysdir = dir.resolve("sys").resolve("block").resolve("zzz").resolve("queue");
+ Files.createDirectories(sysdir);
+ try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
+ o.write("1\n".getBytes(StandardCharsets.US_ASCII));
+ }
+ Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+ FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+
+ Path mockPath = mockLinux.getPath(dir.toString());
+ assertTrue(IOUtils.spinsLinux(mockPath));
+ }
+
+ public void testManyPartitions() throws Exception {
+ assumeFalse("windows is not supported", Constants.WINDOWS);
+ Path dir = createTempDir();
+ dir = FilterPath.unwrap(dir).toRealPath();
+
+ // fake ssd
+ FileStore root = new MockFileStore(Files.getFileStore(dir), dir.toString() + " (/dev/zzz12)");
+ // make a fake /dev/zzz11 for it
+ Path devdir = dir.resolve("dev");
+ Files.createDirectories(devdir);
+ Files.createFile(devdir.resolve("zzz12"));
+ // make a fake /sys/block/zzz/queue/rotational file for it
+ Path sysdir = dir.resolve("sys").resolve("block").resolve("zzz").resolve("queue");
+ Files.createDirectories(sysdir);
+ try (OutputStream o = Files.newOutputStream(sysdir.resolve("rotational"))) {
+ o.write("0\n".getBytes(StandardCharsets.US_ASCII));
+ }
+ Map<String,FileStore> mappings = Collections.singletonMap(dir.toString(), root);
+ FileSystem mockLinux = new MockLinuxFileSystemProvider(dir.getFileSystem(), mappings, dir).getFileSystem(null);
+
+ Path mockPath = mockLinux.getPath(dir.toString());
+ assertFalse(IOUtils.spinsLinux(mockPath));
+ }
+
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java Sun Jan 4 14:53:12 2015
@@ -21,7 +21,7 @@ import org.apache.lucene.codecs.Codec;
* limitations under the License.
*/
-// TODO: maybe we should test this with mocks, but its easy
+// TODO: maybe we should test this with mocks, but it's easy
// enough to test the basics via Codec
public class TestNamedSPILoader extends LuceneTestCase {
@@ -31,7 +31,7 @@ public class TestNamedSPILoader extends
assertEquals(currentName, codec.getName());
}
- // we want an exception if its not found.
+ // we want an exception if it's not found.
public void testBogusLookup() {
try {
Codec.forName("dskfdskfsdfksdfdsf");
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java?rev=1649347&r1=1649346&r2=1649347&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java Sun Jan 4 14:53:12 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.io.Reader;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -355,4 +356,32 @@ public class TestQueryBuilder extends Lu
expected.setSlop(3);
assertEquals(expected, builder.createPhraseQuery("field", "ä¸å½", 3));
}
+
+ public void testNoTermAttribute() {
+ //Can't use MockTokenizer because it adds TermAttribute and we don't want that
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(
+ new Tokenizer() {
+ boolean wasReset = false;
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ assertFalse(wasReset);
+ wasReset = true;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ assertTrue(wasReset);
+ return false;
+ }
+ }
+ );
+ }
+ };
+ QueryBuilder builder = new QueryBuilder(analyzer);
+ assertNull(builder.createBooleanQuery("field", "whatever"));
+ }
}