You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/10/21 20:58:44 UTC
svn commit: r1534320 [19/39] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/lucene/expressions/
dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/
dev-tools/maven/lucene/ dev-tools/maven/lucene/expressions/...
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java Mon Oct 21 18:58:24 2013
@@ -250,12 +250,13 @@ public class TestFieldCache extends Luce
termsIndex = cache.getTermsIndex(reader, "bogusfield");
// getTerms
- BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString");
- assertSame("Second request to cache return same array", terms, cache.getTerms(reader, "theRandomUnicodeString"));
+ BinaryDocValues terms = cache.getTerms(reader, "theRandomUnicodeString", true);
+ assertSame("Second request to cache return same array", terms, cache.getTerms(reader, "theRandomUnicodeString", true));
+ Bits bits = cache.getDocsWithField(reader, "theRandomUnicodeString");
for (int i = 0; i < NUM_DOCS; i++) {
terms.get(i, br);
final BytesRef term;
- if (br.bytes == BinaryDocValues.MISSING) {
+ if (!bits.get(i)) {
term = null;
} else {
term = br;
@@ -265,7 +266,7 @@ public class TestFieldCache extends Luce
}
// test bad field
- terms = cache.getTerms(reader, "bogusfield");
+ terms = cache.getTerms(reader, "bogusfield", false);
// getDocTermOrds
SortedSetDocValues termOrds = cache.getDocTermOrds(reader, "theRandomUnicodeMultiValuedField");
@@ -296,7 +297,7 @@ public class TestFieldCache extends Luce
termOrds = cache.getDocTermOrds(reader, "bogusfield");
assertTrue(termOrds.getValueCount() == 0);
- FieldCache.DEFAULT.purge(reader);
+ FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
}
public void testEmptyIndex() throws Exception {
@@ -305,9 +306,9 @@ public class TestFieldCache extends Luce
writer.close();
IndexReader r = DirectoryReader.open(dir);
AtomicReader reader = SlowCompositeReaderWrapper.wrap(r);
- FieldCache.DEFAULT.getTerms(reader, "foobar");
+ FieldCache.DEFAULT.getTerms(reader, "foobar", true);
FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
- FieldCache.DEFAULT.purge(reader);
+ FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheKey());
r.close();
dir.close();
}
@@ -460,7 +461,7 @@ public class TestFieldCache extends Luce
fail();
} catch (IllegalStateException expected) {}
- BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary");
+ BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true);
binary.get(0, scratch);
assertEquals("binary value", scratch.utf8ToString());
@@ -480,7 +481,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary");
- assertTrue(bits instanceof Bits.MatchAllBits);
+ assertTrue(bits.get(0));
// Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
try {
@@ -493,7 +494,7 @@ public class TestFieldCache extends Luce
fail();
} catch (IllegalStateException expected) {}
- binary = FieldCache.DEFAULT.getTerms(ar, "sorted");
+ binary = FieldCache.DEFAULT.getTerms(ar, "sorted", true);
binary.get(0, scratch);
assertEquals("sorted value", scratch.utf8ToString());
@@ -510,14 +511,14 @@ public class TestFieldCache extends Luce
assertEquals(1, sortedSet.getValueCount());
bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted");
- assertTrue(bits instanceof Bits.MatchAllBits);
+ assertTrue(bits.get(0));
// Numeric type: can be retrieved via getInts() and so on
Ints numeric = FieldCache.DEFAULT.getInts(ar, "numeric", false);
assertEquals(42, numeric.get(0));
try {
- FieldCache.DEFAULT.getTerms(ar, "numeric");
+ FieldCache.DEFAULT.getTerms(ar, "numeric", true);
fail();
} catch (IllegalStateException expected) {}
@@ -537,7 +538,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric");
- assertTrue(bits instanceof Bits.MatchAllBits);
+ assertTrue(bits.get(0));
// SortedSet type: can be retrieved via getDocTermOrds()
if (defaultCodecSupportsSortedSet()) {
@@ -547,7 +548,7 @@ public class TestFieldCache extends Luce
} catch (IllegalStateException expected) {}
try {
- FieldCache.DEFAULT.getTerms(ar, "sortedset");
+ FieldCache.DEFAULT.getTerms(ar, "sortedset", true);
fail();
} catch (IllegalStateException expected) {}
@@ -569,7 +570,7 @@ public class TestFieldCache extends Luce
assertEquals(2, sortedSet.getValueCount());
bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset");
- assertTrue(bits instanceof Bits.MatchAllBits);
+ assertTrue(bits.get(0));
}
ir.close();
@@ -603,14 +604,14 @@ public class TestFieldCache extends Luce
assertEquals(0, doubles.get(0), 0.0D);
BytesRef scratch = new BytesRef();
- BinaryDocValues binaries = cache.getTerms(ar, "bogusterms");
+ BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
binaries.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
assertEquals(-1, sorted.getOrd(0));
sorted.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
sortedSet.setDocument(0);
@@ -662,14 +663,14 @@ public class TestFieldCache extends Luce
assertEquals(0, doubles.get(0), 0.0D);
BytesRef scratch = new BytesRef();
- BinaryDocValues binaries = cache.getTerms(ar, "bogusterms");
+ BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
binaries.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
assertEquals(-1, sorted.getOrd(0));
sorted.get(0, scratch);
- assertTrue(scratch.bytes == BinaryDocValues.MISSING);
+ assertEquals(0, scratch.length);
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued");
sortedSet.setDocument(0);
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java Mon Oct 21 18:58:24 2013
@@ -160,6 +160,58 @@ public class TestMultiTermConstantScore
result[i].score, SCORE_COMP_THRESH);
}
+ result = search.search(csrq("data", "1", "6", T, T, MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT), null, 1000).scoreDocs;
+ numHits = result.length;
+ assertEquals("wrong number of results", 6, numHits);
+ for (int i = 0; i < numHits; i++) {
+ assertEquals("score for " + i + " was not the same", score,
+ result[i].score, SCORE_COMP_THRESH);
+ }
+ }
+
+ @Test // Test for LUCENE-5245: Empty MTQ rewrites should have a consistent norm, so always need to return a CSQ!
+ public void testEqualScoresWhenNoHits() throws IOException {
+ // NOTE: uses index build in *this* setUp
+
+ IndexSearcher search = newSearcher(reader);
+
+ ScoreDoc[] result;
+
+ TermQuery dummyTerm = new TermQuery(new Term("data", "1"));
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(dummyTerm, BooleanClause.Occur.SHOULD); // hits one doc
+ bq.add(csrq("data", "#", "#", T, T), BooleanClause.Occur.SHOULD); // hits no docs
+ result = search.search(bq, null, 1000).scoreDocs;
+ int numHits = result.length;
+ assertEquals("wrong number of results", 1, numHits);
+ float score = result[0].score;
+ for (int i = 1; i < numHits; i++) {
+ assertEquals("score for " + i + " was not the same", score,
+ result[i].score, SCORE_COMP_THRESH);
+ }
+
+ bq = new BooleanQuery();
+ bq.add(dummyTerm, BooleanClause.Occur.SHOULD); // hits one doc
+ bq.add(csrq("data", "#", "#", T, T, MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE), BooleanClause.Occur.SHOULD); // hits no docs
+ result = search.search(bq, null, 1000).scoreDocs;
+ numHits = result.length;
+ assertEquals("wrong number of results", 1, numHits);
+ for (int i = 0; i < numHits; i++) {
+ assertEquals("score for " + i + " was not the same", score,
+ result[i].score, SCORE_COMP_THRESH);
+ }
+
+ bq = new BooleanQuery();
+ bq.add(dummyTerm, BooleanClause.Occur.SHOULD); // hits one doc
+ bq.add(csrq("data", "#", "#", T, T, MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT), BooleanClause.Occur.SHOULD); // hits no docs
+ result = search.search(bq, null, 1000).scoreDocs;
+ numHits = result.length;
+ assertEquals("wrong number of results", 1, numHits);
+ for (int i = 0; i < numHits; i++) {
+ assertEquals("score for " + i + " was not the same", score,
+ result[i].score, SCORE_COMP_THRESH);
+ }
}
@Test
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java Mon Oct 21 18:58:24 2013
@@ -617,16 +617,16 @@ public class TestPhraseQuery extends Luc
break;
}
}
- TokenStream ts = analyzer.tokenStream("ignore", term);
- CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while(ts.incrementToken()) {
- String text = termAttr.toString();
- doc.add(text);
- sb.append(text).append(' ');
+ try (TokenStream ts = analyzer.tokenStream("ignore", term)) {
+ CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
+ while(ts.incrementToken()) {
+ String text = termAttr.toString();
+ doc.add(text);
+ sb.append(text).append(' ');
+ }
+ ts.end();
}
- ts.end();
- ts.close();
} else {
// pick existing sub-phrase
List<String> lastDoc = docs.get(r.nextInt(docs.size()));
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSearcherManager.java Mon Oct 21 18:58:24 2013
@@ -105,6 +105,10 @@ public class TestSearcherManager extends
@Override
public void run() {
try {
+ if (VERBOSE) {
+ System.out.println("[" + Thread.currentThread().getName() + "]: launch reopen thread");
+ }
+
while(System.currentTimeMillis() < stopTime) {
Thread.sleep(_TestUtil.nextInt(random(), 1, 100));
writer.commit();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java Mon Oct 21 18:58:24 2013
@@ -76,7 +76,7 @@ public class TestSimilarityProvider exte
public void testBasics() throws Exception {
// sanity check of norms writer
// TODO: generalize
- AtomicReader slow = new SlowCompositeReaderWrapper(reader);
+ AtomicReader slow = SlowCompositeReaderWrapper.wrap(reader);
NumericDocValues fooNorms = slow.getNormValues("foo");
NumericDocValues barNorms = slow.getNormValues("bar");
for (int i = 0; i < slow.maxDoc(); i++) {
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java Mon Oct 21 18:58:24 2013
@@ -31,10 +31,12 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
/** Tests basic sorting on docvalues fields.
* These are mostly like TestSort's tests, except each test
* indexes the field up-front as docvalues, and checks no fieldcaches were made */
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42"}) // avoid codecs that don't support "missing"
public class TestSortDocValues extends LuceneTestCase {
@Override
@@ -291,6 +293,70 @@ public class TestSortDocValues extends L
dir.close();
}
+ /** Tests sorting on type int with a missing value */
+ public void testIntMissing() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", -1));
+ doc.add(newStringField("value", "-1", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", 4));
+ doc.add(newStringField("value", "4", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ Sort sort = new Sort(new SortField("value", SortField.Type.INT));
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(3, td.totalHits);
+ // null is treated as a 0
+ assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
+ /** Tests sorting on type int, specifying the missing value should be treated as Integer.MAX_VALUE */
+ public void testIntMissingLast() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", -1));
+ doc.add(newStringField("value", "-1", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", 4));
+ doc.add(newStringField("value", "4", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ SortField sortField = new SortField("value", SortField.Type.INT);
+ sortField.setMissingValue(Integer.MAX_VALUE);
+ Sort sort = new Sort(sortField);
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(3, td.totalHits);
+ // null is treated as a Integer.MAX_VALUE
+ assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
/** Tests sorting on type long */
public void testLong() throws IOException {
Directory dir = newDirectory();
@@ -359,6 +425,70 @@ public class TestSortDocValues extends L
dir.close();
}
+ /** Tests sorting on type long with a missing value */
+ public void testLongMissing() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", -1));
+ doc.add(newStringField("value", "-1", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", 4));
+ doc.add(newStringField("value", "4", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ Sort sort = new Sort(new SortField("value", SortField.Type.LONG));
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(3, td.totalHits);
+ // null is treated as 0
+ assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
+ /** Tests sorting on type long, specifying the missing value should be treated as Long.MAX_VALUE */
+ public void testLongMissingLast() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", -1));
+ doc.add(newStringField("value", "-1", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("value", 4));
+ doc.add(newStringField("value", "4", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ SortField sortField = new SortField("value", SortField.Type.LONG);
+ sortField.setMissingValue(Long.MAX_VALUE);
+ Sort sort = new Sort(sortField);
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(3, td.totalHits);
+ // null is treated as Long.MAX_VALUE
+ assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
/** Tests sorting on type float */
public void testFloat() throws IOException {
Directory dir = newDirectory();
@@ -427,6 +557,70 @@ public class TestSortDocValues extends L
dir.close();
}
+ /** Tests sorting on type float with a missing value */
+ public void testFloatMissing() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("value", -1.3F));
+ doc.add(newStringField("value", "-1.3", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("value", 4.2F));
+ doc.add(newStringField("value", "4.2", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT));
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(3, td.totalHits);
+ // null is treated as 0
+ assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertEquals("4.2", searcher.doc(td.scoreDocs[2].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
+ /** Tests sorting on type float, specifying the missing value should be treated as Float.MAX_VALUE */
+ public void testFloatMissingLast() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("value", -1.3F));
+ doc.add(newStringField("value", "-1.3", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("value", 4.2F));
+ doc.add(newStringField("value", "4.2", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ SortField sortField = new SortField("value", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.MAX_VALUE);
+ Sort sort = new Sort(sortField);
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(3, td.totalHits);
+ // null is treated as Float.MAX_VALUE
+ assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
/** Tests sorting on type double */
public void testDouble() throws IOException {
Directory dir = newDirectory();
@@ -533,4 +727,78 @@ public class TestSortDocValues extends L
ir.close();
dir.close();
}
+
+ /** Tests sorting on type double with a missing value */
+ public void testDoubleMissing() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("value", -1.3));
+ doc.add(newStringField("value", "-1.3", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("value", 4.2333333333333));
+ doc.add(newStringField("value", "4.2333333333333", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("value", 4.2333333333332));
+ doc.add(newStringField("value", "4.2333333333332", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE));
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(4, td.totalHits);
+ // null treated as a 0
+ assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value"));
+ assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[3].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
+
+ /** Tests sorting on type double, specifying the missing value should be treated as Double.MAX_VALUE */
+ public void testDoubleMissingLast() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("value", -1.3));
+ doc.add(newStringField("value", "-1.3", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("value", 4.2333333333333));
+ doc.add(newStringField("value", "4.2333333333333", Field.Store.YES));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("value", 4.2333333333332));
+ doc.add(newStringField("value", "4.2333333333332", Field.Store.YES));
+ writer.addDocument(doc);
+ IndexReader ir = writer.getReader();
+ writer.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ SortField sortField = new SortField("value", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.MAX_VALUE);
+ Sort sort = new Sort(sortField);
+
+ TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
+ assertEquals(4, td.totalHits);
+ // null treated as Double.MAX_VALUE
+ assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value"));
+ assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value"));
+ assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value"));
+ assertNull(searcher.doc(td.scoreDocs[3].doc).get("value"));
+
+ ir.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java Mon Oct 21 18:58:24 2013
@@ -227,7 +227,8 @@ public class TestTermRangeQuery extends
}
@Override
- public void reset() throws IOException {;
+ public void reset() throws IOException {
+ super.reset();
done = false;
}
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java Mon Oct 21 18:58:24 2013
@@ -59,7 +59,7 @@ public class PayloadHelper {
public final class PayloadAnalyzer extends Analyzer {
public PayloadAnalyzer() {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
}
@Override
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Mon Oct 21 18:58:24 2013
@@ -64,7 +64,7 @@ public class TestPayloadTermQuery extend
private static class PayloadAnalyzer extends Analyzer {
private PayloadAnalyzer() {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
}
@Override
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java Mon Oct 21 18:58:24 2013
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -572,4 +573,20 @@ public class TestSimilarityBase extends
dir.close();
super.tearDown();
}
+
+ // LUCENE-5221
+ public void testDiscountOverlapsBoost() throws IOException {
+ DefaultSimilarity expected = new DefaultSimilarity();
+ SimilarityBase actual = new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());
+ expected.setDiscountOverlaps(false);
+ actual.setDiscountOverlaps(false);
+ FieldInvertState state = new FieldInvertState("foo");
+ state.setLength(5);
+ state.setNumOverlap(2);
+ state.setBoost(3);
+ assertEquals(expected.computeNorm(state), actual.computeNorm(state));
+ expected.setDiscountOverlaps(true);
+ actual.setDiscountOverlaps(true);
+ assertEquals(expected.computeNorm(state), actual.computeNorm(state));
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java Mon Oct 21 18:58:24 2013
@@ -283,7 +283,7 @@ public class TestBufferedIndexInput exte
}
}
- private static class MockFSDirectory extends Directory {
+ private static class MockFSDirectory extends BaseDirectory {
List<IndexInput> allIndexInputs = new ArrayList<IndexInput>();
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java Mon Oct 21 18:58:24 2013
@@ -20,7 +20,7 @@ package org.apache.lucene.util;
public class TestBroadWord extends LuceneTestCase {
private void tstRank(long x) {
- assertEquals("rank9(" + x + ")", Long.bitCount(x), BroadWord.rank9(x));
+ assertEquals("rank(" + x + ")", Long.bitCount(x), BroadWord.bitCount(x));
}
public void testRank1() {
@@ -34,7 +34,7 @@ public class TestBroadWord extends Lucen
private void tstSelect(long x, int r, int exp) {
assertEquals("selectNaive(" + x + "," + r + ")", exp, BroadWord.selectNaive(x, r));
- assertEquals("select9(" + x + "," + r + ")", exp, BroadWord.select9(x, r));
+ assertEquals("select(" + x + "," + r + ")", exp, BroadWord.select(x, r));
}
public void testSelectFromZero() {
@@ -77,7 +77,7 @@ public class TestBroadWord extends Lucen
public void testPerfSelectAllBitsBroad() {
for (int j = 0; j < 100000; j++) { // 1000000 for real perf test
for (int i = 0; i < 64; i++) {
- assertEquals(i, BroadWord.select9(0xFFFFFFFFFFFFFFFFL, i+1));
+ assertEquals(i, BroadWord.select(0xFFFFFFFFFFFFFFFFL, i+1));
}
}
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java Mon Oct 21 18:58:24 2013
@@ -120,7 +120,7 @@ public class TestFieldCacheSanityChecker
cache.purgeAllCaches();
cache.getInts(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
- cache.getTerms(readerX, "theInt");
+ cache.getTerms(readerX, "theInt", false);
// // //
@@ -142,9 +142,9 @@ public class TestFieldCacheSanityChecker
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
- cache.getTerms(readerA, "theInt");
- cache.getTerms(readerB, "theInt");
- cache.getTerms(readerX, "theInt");
+ cache.getTerms(readerA, "theInt", false);
+ cache.getTerms(readerB, "theInt", false);
+ cache.getTerms(readerX, "theInt", false);
// // //
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestFixedBitSet.java Mon Oct 21 18:58:24 2013
@@ -328,7 +328,50 @@ public class TestFixedBitSet extends Bas
checkNextSetBitArray(new int[0], setBits.length + random().nextInt(10));
}
-}
-
+
+ public void testGrow() {
+ FixedBitSet bits = new FixedBitSet(5);
+ bits.set(1);
+ bits.set(4);
+
+ FixedBitSet newBits = new FixedBitSet(bits, 8); // grow within the word
+ assertTrue(newBits.get(1));
+ assertTrue(newBits.get(4));
+ newBits = new FixedBitSet(bits, 72); // grow beyond one word
+ assertTrue(newBits.get(1));
+ assertTrue(newBits.get(4));
+ }
+
+ public void testShrink() {
+ FixedBitSet bits = new FixedBitSet(72);
+ bits.set(1);
+ bits.set(4);
+ bits.set(69);
+
+ FixedBitSet newBits = new FixedBitSet(bits, 66); // shrink within the word
+ assertTrue(newBits.get(1));
+ assertTrue(newBits.get(4));
+ boolean hitError = true;
+ try {
+ newBits.get(69);
+ hitError = false;
+ } catch (AssertionError e) {
+ hitError = true;
+ }
+ assertTrue(hitError);
+ newBits = new FixedBitSet(bits, 8); // shrink beyond one word
+ assertTrue(newBits.get(1));
+ assertTrue(newBits.get(4));
+ hitError = true;
+ try {
+ newBits.get(69);
+ hitError = false;
+ } catch (AssertionError e) {
+ hitError = true;
+ }
+ assertTrue(hitError);
+ }
+
+}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java Mon Oct 21 18:58:24 2013
@@ -102,4 +102,80 @@ public class TestMathUtil extends Lucene
assertEquals(Long.MIN_VALUE, MathUtil.gcd(Long.MIN_VALUE, Long.MIN_VALUE));
}
+ public void testAcoshMethod() {
+ // acosh(NaN) == NaN
+ assertTrue(Double.isNaN(MathUtil.acosh(Double.NaN)));
+ // acosh(1) == +0
+ assertEquals(0, Double.doubleToLongBits(MathUtil.acosh(1D)));
+ // acosh(POSITIVE_INFINITY) == POSITIVE_INFINITY
+ assertEquals(Double.doubleToLongBits(Double.POSITIVE_INFINITY),
+ Double.doubleToLongBits(MathUtil.acosh(Double.POSITIVE_INFINITY)));
+ // acosh(x) : x < 1 == NaN
+ assertTrue(Double.isNaN(MathUtil.acosh(0.9D))); // x < 1
+ assertTrue(Double.isNaN(MathUtil.acosh(0D))); // x == 0
+ assertTrue(Double.isNaN(MathUtil.acosh(-0D))); // x == -0
+ assertTrue(Double.isNaN(MathUtil.acosh(-0.9D))); // x < 0
+ assertTrue(Double.isNaN(MathUtil.acosh(-1D))); // x == -1
+ assertTrue(Double.isNaN(MathUtil.acosh(-10D))); // x < -1
+ assertTrue(Double.isNaN(MathUtil.acosh(Double.NEGATIVE_INFINITY))); // x == -Inf
+
+ double epsilon = 0.000001;
+ assertEquals(0, MathUtil.acosh(1), epsilon);
+ assertEquals(1.5667992369724109, MathUtil.acosh(2.5), epsilon);
+ assertEquals(14.719378760739708, MathUtil.acosh(1234567.89), epsilon);
+ }
+
+ public void testAsinhMethod() {
+
+ // asinh(NaN) == NaN
+ assertTrue(Double.isNaN(MathUtil.asinh(Double.NaN)));
+ // asinh(+0) == +0
+ assertEquals(0, Double.doubleToLongBits(MathUtil.asinh(0D)));
+ // asinh(-0) == -0
+ assertEquals(Double.doubleToLongBits(-0D), Double.doubleToLongBits(MathUtil.asinh(-0D)));
+ // asinh(POSITIVE_INFINITY) == POSITIVE_INFINITY
+ assertEquals(Double.doubleToLongBits(Double.POSITIVE_INFINITY),
+ Double.doubleToLongBits(MathUtil.asinh(Double.POSITIVE_INFINITY)));
+ // asinh(NEGATIVE_INFINITY) == NEGATIVE_INFINITY
+ assertEquals(Double.doubleToLongBits(Double.NEGATIVE_INFINITY),
+ Double.doubleToLongBits(MathUtil.asinh(Double.NEGATIVE_INFINITY)));
+
+ double epsilon = 0.000001;
+ assertEquals(-14.719378760740035, MathUtil.asinh(-1234567.89), epsilon);
+ assertEquals(-1.6472311463710958, MathUtil.asinh(-2.5), epsilon);
+ assertEquals(-0.8813735870195429, MathUtil.asinh(-1), epsilon);
+ assertEquals(0, MathUtil.asinh(0), 0);
+ assertEquals(0.8813735870195429, MathUtil.asinh(1), epsilon);
+ assertEquals(1.6472311463710958, MathUtil.asinh(2.5), epsilon);
+ assertEquals(14.719378760740035, MathUtil.asinh(1234567.89), epsilon );
+ }
+
+ public void testAtanhMethod() {
+ // atanh(NaN) == NaN
+ assertTrue(Double.isNaN(MathUtil.atanh(Double.NaN)));
+ // atanh(+0) == +0
+ assertEquals(0, Double.doubleToLongBits(MathUtil.atanh(0D)));
+ // atanh(-0) == -0
+ assertEquals(Double.doubleToLongBits(-0D),
+ Double.doubleToLongBits(MathUtil.atanh(-0D)));
+ // atanh(1) == POSITIVE_INFINITY
+ assertEquals(Double.doubleToLongBits(Double.POSITIVE_INFINITY),
+ Double.doubleToLongBits(MathUtil.atanh(1D)));
+ // atanh(-1) == NEGATIVE_INFINITY
+ assertEquals(Double.doubleToLongBits(Double.NEGATIVE_INFINITY),
+ Double.doubleToLongBits(MathUtil.atanh(-1D)));
+ // atanh(x) : Math.abs(x) > 1 == NaN
+ assertTrue(Double.isNaN(MathUtil.atanh(1.1D))); // x > 1
+ assertTrue(Double.isNaN(MathUtil.atanh(Double.POSITIVE_INFINITY))); // x == Inf
+ assertTrue(Double.isNaN(MathUtil.atanh(-1.1D))); // x < -1
+ assertTrue(Double.isNaN(MathUtil.atanh(Double.NEGATIVE_INFINITY))); // x == -Inf
+
+ double epsilon = 0.000001;
+ assertEquals(Double.NEGATIVE_INFINITY, MathUtil.atanh(-1), 0);
+ assertEquals(-0.5493061443340549, MathUtil.atanh(-0.5), epsilon);
+ assertEquals(0, MathUtil.atanh(0), 0);
+ assertEquals(0.5493061443340549, MathUtil.atanh(0.5), epsilon);
+ assertEquals(Double.POSITIVE_INFINITY, MathUtil.atanh(1), 0);
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java Mon Oct 21 18:58:24 2013
@@ -24,21 +24,22 @@ import org.apache.lucene.codecs.Codec;
// TODO: maybe we should test this with mocks, but its easy
// enough to test the basics via Codec
public class TestNamedSPILoader extends LuceneTestCase {
+
public void testLookup() {
- Codec codec = Codec.forName("Lucene42");
- assertEquals("Lucene42", codec.getName());
+ Codec codec = Codec.forName("Lucene46");
+ assertEquals("Lucene46", codec.getName());
}
// we want an exception if its not found.
public void testBogusLookup() {
try {
- Codec codec = Codec.forName("dskfdskfsdfksdfdsf");
+ Codec.forName("dskfdskfsdfksdfdsf");
fail();
} catch (IllegalArgumentException expected) {}
}
public void testAvailableServices() {
Set<String> codecs = Codec.availableCodecs();
- assertTrue(codecs.contains("Lucene42"));
+ assertTrue(codecs.contains("Lucene46"));
}
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestOpenBitSet.java Mon Oct 21 18:58:24 2013
@@ -331,6 +331,38 @@ public class TestOpenBitSet extends Base
checkPrevSetBitArray(new int[] {0,2});
}
+ public void testEnsureCapacity() {
+ OpenBitSet bits = new OpenBitSet(1);
+ int bit = random().nextInt(100) + 10;
+ bits.ensureCapacity(bit); // make room for more bits
+ bits.fastSet(bit-1);
+ assertTrue(bits.fastGet(bit-1));
+ bits.ensureCapacity(bit + 1);
+ bits.fastSet(bit);
+ assertTrue(bits.fastGet(bit));
+ bits.ensureCapacity(3); // should not change numBits nor grow the array
+ bits.fastSet(3);
+ assertTrue(bits.fastGet(3));
+ bits.fastSet(bit-1);
+ assertTrue(bits.fastGet(bit-1));
+
+ // test ensureCapacityWords
+ int numWords = random().nextInt(10) + 2; // make sure we grow the array (at least 128 bits)
+ bits.ensureCapacityWords(numWords);
+ bit = _TestUtil.nextInt(random(), 127, (numWords << 6)-1); // pick a bit >= to 128, but still within range
+ bits.fastSet(bit);
+ assertTrue(bits.fastGet(bit));
+ bits.fastClear(bit);
+ assertFalse(bits.fastGet(bit));
+ bits.fastFlip(bit);
+ assertTrue(bits.fastGet(bit));
+ bits.ensureCapacityWords(2); // should not change numBits nor grow the array
+ bits.fastSet(3);
+ assertTrue(bits.fastGet(3));
+ bits.fastSet(bit-1);
+ assertTrue(bits.fastGet(bit-1));
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java Mon Oct 21 18:58:24 2013
@@ -22,6 +22,7 @@ import java.util.*;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -30,6 +31,9 @@ import org.junit.Ignore;
public class TestPagedBytes extends LuceneTestCase {
+ // Writes random byte/s to "normal" file in dir, then
+ // copies into PagedBytes and verifies with
+ // PagedBytes.Reader:
public void testDataInputOutput() throws Exception {
Random random = random();
for(int iter=0;iter<5*RANDOM_MULTIPLIER;iter++) {
@@ -90,6 +94,60 @@ public class TestPagedBytes extends Luce
}
}
+ // Writes random byte/s into PagedBytes via
+ // .getDataOutput(), then verifies with
+ // PagedBytes.getDataInput():
+ public void testDataInputOutput2() throws Exception {
+ Random random = random();
+ for(int iter=0;iter<5*RANDOM_MULTIPLIER;iter++) {
+ final int blockBits = _TestUtil.nextInt(random, 1, 20);
+ final int blockSize = 1 << blockBits;
+ final PagedBytes p = new PagedBytes(blockBits);
+ final DataOutput out = p.getDataOutput();
+ final int numBytes = random().nextInt(10000000);
+
+ final byte[] answer = new byte[numBytes];
+ random().nextBytes(answer);
+ int written = 0;
+ while(written < numBytes) {
+ if (random().nextInt(10) == 7) {
+ out.writeByte(answer[written++]);
+ } else {
+ int chunk = Math.min(random().nextInt(1000), numBytes - written);
+ out.writeBytes(answer, written, chunk);
+ written += chunk;
+ }
+ }
+
+ final PagedBytes.Reader reader = p.freeze(random.nextBoolean());
+
+ final DataInput in = p.getDataInput();
+
+ final byte[] verify = new byte[numBytes];
+ int read = 0;
+ while(read < numBytes) {
+ if (random().nextInt(10) == 7) {
+ verify[read++] = in.readByte();
+ } else {
+ int chunk = Math.min(random().nextInt(1000), numBytes - read);
+ in.readBytes(verify, read, chunk);
+ read += chunk;
+ }
+ }
+ assertTrue(Arrays.equals(answer, verify));
+
+ final BytesRef slice = new BytesRef();
+ for(int iter2=0;iter2<100;iter2++) {
+ final int pos = random.nextInt(numBytes-1);
+ final int len = random.nextInt(Math.min(blockSize+1, numBytes - pos));
+ reader.fillSlice(slice, pos, len);
+ for(int byteUpto=0;byteUpto<len;byteUpto++) {
+ assertEquals(answer[pos + byteUpto], slice.bytes[slice.offset + byteUpto]);
+ }
+ }
+ }
+ }
+
@Ignore // memory hole
public void testOverflow() throws IOException {
BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("testOverflow"));
@@ -126,4 +184,5 @@ public class TestPagedBytes extends Luce
in.close();
dir.close();
}
+
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java Mon Oct 21 18:58:24 2013
@@ -24,6 +24,8 @@ import org.junit.runner.JUnitCore;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+
public class TestFailIfDirectoryNotClosed extends WithNestedTests {
public TestFailIfDirectoryNotClosed() {
super(true);
@@ -39,6 +41,9 @@ public class TestFailIfDirectoryNotClose
@Test
public void testFailIfDirectoryNotClosed() {
Result r = JUnitCore.runClasses(Nested1.class);
+ RandomizedTest.assumeTrue("Ignoring nested test, very likely zombie threads present.",
+ r.getIgnoreCount() == 0);
+
for (Failure f : r.getFailures()) {
System.out.println("Failure: " + f);
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java Mon Oct 21 18:58:24 2013
@@ -29,6 +29,9 @@ import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.JUnitCore;
import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
// LUCENE-4456: Test that we fail if there are unreferenced files
public class TestFailIfUnreferencedFiles extends WithNestedTests {
@@ -53,6 +56,17 @@ public class TestFailIfUnreferencedFiles
@Test
public void testFailIfUnreferencedFiles() {
Result r = JUnitCore.runClasses(Nested1.class);
- Assert.assertEquals(1, r.getFailureCount());
+ RandomizedTest.assumeTrue("Ignoring nested test, very likely zombie threads present.",
+ r.getIgnoreCount() == 0);
+
+ // We are suppressing output anyway so dump the failures.
+ for (Failure f : r.getFailures()) {
+ System.out.println(f.getTrace());
+ }
+
+ Assert.assertEquals("Expected exactly one failure.",
+ 1, r.getFailureCount());
+ Assert.assertTrue("Expected unreferenced files assertion.",
+ r.getFailures().get(0).getTrace().contains("unreferenced files:"));
}
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailOnFieldCacheInsanity.java Mon Oct 21 18:58:24 2013
@@ -56,7 +56,7 @@ public class TestFailOnFieldCacheInsanit
public void testDummy() throws Exception {
makeIndex();
assertNotNull(FieldCache.DEFAULT.getTermsIndex(subR, "ints"));
- assertNotNull(FieldCache.DEFAULT.getTerms(subR, "ints"));
+ assertNotNull(FieldCache.DEFAULT.getTerms(subR, "ints", false));
// NOTE: do not close reader/directory, else it
// purges FC entries
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestEliasFanoSequence.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestEliasFanoSequence.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestEliasFanoSequence.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestEliasFanoSequence.java Mon Oct 21 18:58:24 2013
@@ -21,13 +21,13 @@ import org.apache.lucene.util.LuceneTest
public class TestEliasFanoSequence extends LuceneTestCase {
- private static EliasFanoEncoder makeEncoder(long[] values) {
+ private static EliasFanoEncoder makeEncoder(long[] values, long indexInterval) {
long upperBound = -1L;
for (long value: values) {
assertTrue(value >= upperBound); // test data ok
upperBound = value;
}
- EliasFanoEncoder efEncoder = new EliasFanoEncoder(values.length, upperBound);
+ EliasFanoEncoder efEncoder = new EliasFanoEncoder(values.length, upperBound, indexInterval);
for (long value: values) {
efEncoder.encodeNext(value);
}
@@ -64,13 +64,13 @@ public class TestEliasFanoSequence exten
long advanceValue = efd.advanceToValue(expValue);
assertFalse("advanceValue at end too early", EliasFanoDecoder.NO_MORE_VALUES == advanceValue);
assertEquals(expValue, advanceValue);
- assertEquals(index, efd.index());
+ assertEquals(index, efd.currentIndex());
previousValue = expValue;
}
index++;
}
long advanceValue = efd.advanceToValue(previousValue+1);
- assertEquals(EliasFanoDecoder.NO_MORE_VALUES, advanceValue);
+ assertEquals("at end", EliasFanoDecoder.NO_MORE_VALUES, advanceValue);
}
private static void tstDecodeAdvanceToMultiples(long[] values, EliasFanoDecoder efd, final long m) {
@@ -86,7 +86,7 @@ public class TestEliasFanoSequence exten
long advanceValue = efd.advanceToValue(mm);
assertFalse("advanceValue at end too early", EliasFanoDecoder.NO_MORE_VALUES == advanceValue);
assertEquals(expValue, advanceValue);
- assertEquals(index, efd.index());
+ assertEquals(index, efd.currentIndex());
previousValue = expValue;
do {
mm += m;
@@ -118,7 +118,7 @@ public class TestEliasFanoSequence exten
long backValue = efd.backToValue(mm);
assertFalse("backToValue at end too early", EliasFanoDecoder.NO_MORE_VALUES == backValue);
assertEquals(expValue, backValue);
- assertEquals(index, efd.index());
+ assertEquals(index, efd.currentIndex());
previousValue = expValue;
do {
mm -= m;
@@ -146,25 +146,31 @@ public class TestEliasFanoSequence exten
}
private static void tstEFS(long[] values, long[] expHighLongs, long[] expLowLongs) {
- EliasFanoEncoder efEncoder = makeEncoder(values);
+ EliasFanoEncoder efEncoder = makeEncoder(values, EliasFanoEncoder.DEFAULT_INDEX_INTERVAL);
tstEqual("upperBits", expHighLongs, efEncoder.getUpperBits());
tstEqual("lowerBits", expLowLongs, efEncoder.getLowerBits());
tstDecodeAll(efEncoder, values);
}
private static void tstEFS2(long[] values) {
- EliasFanoEncoder efEncoder = makeEncoder(values);
+ EliasFanoEncoder efEncoder = makeEncoder(values, EliasFanoEncoder.DEFAULT_INDEX_INTERVAL);
tstDecodeAll(efEncoder, values);
}
private static void tstEFSadvanceToAndBackToMultiples(long[] values, long maxValue, long minAdvanceMultiple) {
- EliasFanoEncoder efEncoder = makeEncoder(values);
+ EliasFanoEncoder efEncoder = makeEncoder(values, EliasFanoEncoder.DEFAULT_INDEX_INTERVAL);
for (long m = minAdvanceMultiple; m <= maxValue; m += 1) {
tstDecodeAdvanceToMultiples(values, efEncoder.getDecoder(), m);
tstDecodeBackToMultiples(values, efEncoder.getDecoder(), m);
}
}
+ private EliasFanoEncoder tstEFVI(long[] values, long indexInterval, long[] expIndexBits) {
+ EliasFanoEncoder efEncVI = makeEncoder(values, indexInterval);
+ tstEqual("upperZeroBitPositionIndex", expIndexBits, efEncVI.getIndexBits());
+ return efEncVI;
+ }
+
public void testEmpty() {
long[] values = new long[0];
long[] expHighBits = new long[0];
@@ -223,29 +229,31 @@ public class TestEliasFanoSequence exten
public void testHashCodeEquals() {
long[] values = new long[] {5,8,8,15,32};
- EliasFanoEncoder efEncoder1 = makeEncoder(values);
- EliasFanoEncoder efEncoder2 = makeEncoder(values);
+ EliasFanoEncoder efEncoder1 = makeEncoder(values, EliasFanoEncoder.DEFAULT_INDEX_INTERVAL);
+ EliasFanoEncoder efEncoder2 = makeEncoder(values, EliasFanoEncoder.DEFAULT_INDEX_INTERVAL);
assertEquals(efEncoder1, efEncoder2);
assertEquals(efEncoder1.hashCode(), efEncoder2.hashCode());
- EliasFanoEncoder efEncoder3 = makeEncoder(new long[] {1,2,3});
+ EliasFanoEncoder efEncoder3 = makeEncoder(new long[] {1,2,3}, EliasFanoEncoder.DEFAULT_INDEX_INTERVAL);
assertFalse(efEncoder1.equals(efEncoder3));
assertFalse(efEncoder3.equals(efEncoder1));
assertFalse(efEncoder1.hashCode() == efEncoder3.hashCode()); // implementation ok for these.
}
public void testMonotoneSequences() {
- for (int s = 2; s < 1222; s++) {
+ //for (int s = 2; s < 1222; s++) {
+ for (int s = 2; s < 4422; s++) {
long[] values = new long[s];
for (int i = 0; i < s; i++) {
- values[i] = (i/2);
+ values[i] = (i/2); // upperbound smaller than number of values, only upper bits encoded
}
tstEFS2(values);
}
}
public void testStrictMonotoneSequences() {
- for (int s = 2; s < 1222; s++) {
+ // for (int s = 2; s < 1222; s++) {
+ for (int s = 2; s < 4422; s++) {
long[] values = new long[s];
for (int i = 0; i < s; i++) {
values[i] = i * ((long) i - 1) / 2; // Add a gap of (s-1) to previous
@@ -277,5 +285,98 @@ public class TestEliasFanoSequence exten
tstEFSadvanceToAndBackToMultiples(values, values[s-1], 10);
}
}
+
+ public void testEmptyIndex() {
+ long indexInterval = 2;
+ long[] emptyLongs = new long[0];
+ tstEFVI(emptyLongs, indexInterval, emptyLongs);
+ }
+ public void testMaxContentEmptyIndex() {
+ long indexInterval = 2;
+ long[] twoLongs = new long[] {0,1};
+ long[] emptyLongs = new long[0];
+ tstEFVI(twoLongs, indexInterval, emptyLongs);
+ }
+
+ public void testMinContentNonEmptyIndex() {
+ long indexInterval = 2;
+ long[] twoLongs = new long[] {0,2};
+ long[] indexLongs = new long[] {3}; // high bits 1001, index position after zero bit.
+ tstEFVI(twoLongs, indexInterval, indexLongs);
+ }
+
+ public void testIndexAdvanceToLast() {
+ long indexInterval = 2;
+ long[] twoLongs = new long[] {0,2};
+ long[] indexLongs = new long[] {3}; // high bits 1001
+ EliasFanoEncoder efEncVI = tstEFVI(twoLongs, indexInterval, indexLongs);
+ assertEquals(2, efEncVI.getDecoder().advanceToValue(2));
+ }
+
+ public void testIndexAdvanceToAfterLast() {
+ long indexInterval = 2;
+ long[] twoLongs = new long[] {0,2};
+ long[] indexLongs = new long[] {3}; // high bits 1001
+ EliasFanoEncoder efEncVI = tstEFVI(twoLongs, indexInterval, indexLongs);
+ assertEquals(EliasFanoDecoder.NO_MORE_VALUES, efEncVI.getDecoder().advanceToValue(3));
+ }
+
+ public void testIndexAdvanceToFirst() {
+ long indexInterval = 2;
+ long[] twoLongs = new long[] {0,2};
+ long[] indexLongs = new long[] {3}; // high bits 1001
+ EliasFanoEncoder efEncVI = tstEFVI(twoLongs, indexInterval, indexLongs);
+ assertEquals(0, efEncVI.getDecoder().advanceToValue(0));
+ }
+
+ public void testTwoIndexEntries() {
+ long indexInterval = 2;
+ long[] twoLongs = new long[] {0,1,2,3,4,5};
+ long[] indexLongs = new long[] {4 + 8*16}; // high bits 0b10101010101
+ EliasFanoEncoder efEncVI = tstEFVI(twoLongs, indexInterval, indexLongs);
+ EliasFanoDecoder efDecVI = efEncVI.getDecoder();
+ assertEquals("advance 0", 0, efDecVI.advanceToValue(0));
+ assertEquals("advance 5", 5, efDecVI.advanceToValue(5));
+ assertEquals("advance 6", EliasFanoDecoder.NO_MORE_VALUES, efDecVI.advanceToValue(5));
+ }
+
+ public void testExample2a() { // Figure 2 from Vigna 2012 paper
+ long indexInterval = 4;
+ long[] values = new long[] {5,8,8,15,32}; // two low bits, high values 1,2,2,3,8.
+ long[] indexLongs = new long[] {8 + 12*16}; // high bits 0b 0001 0000 0101 1010
+ EliasFanoEncoder efEncVI = tstEFVI(values, indexInterval, indexLongs);
+ EliasFanoDecoder efDecVI = efEncVI.getDecoder();
+ assertEquals("advance 22", 32, efDecVI.advanceToValue(22));
+ }
+
+ public void testExample2b() { // Figure 2 from Vigna 2012 paper
+ long indexInterval = 4;
+ long[] values = new long[] {5,8,8,15,32}; // two low bits, high values 1,2,2,3,8.
+ long[] indexLongs = new long[] {8 + 12*16}; // high bits 0b 0001 0000 0101 1010
+ EliasFanoEncoder efEncVI = tstEFVI(values, indexInterval, indexLongs);
+ EliasFanoDecoder efDecVI = efEncVI.getDecoder();
+ assertEquals("initial next", 5, efDecVI.nextValue());
+ assertEquals("advance 22", 32, efDecVI.advanceToValue(22));
+ }
+
+ public void testExample2NoIndex1() { // Figure 2 from Vigna 2012 paper, no index, test broadword selection.
+ long indexInterval = 16;
+ long[] values = new long[] {5,8,8,15,32}; // two low bits, high values 1,2,2,3,8.
+ long[] indexLongs = new long[0]; // high bits 0b 0001 0000 0101 1010
+ EliasFanoEncoder efEncVI = tstEFVI(values, indexInterval, indexLongs);
+ EliasFanoDecoder efDecVI = efEncVI.getDecoder();
+ assertEquals("advance 22", 32, efDecVI.advanceToValue(22));
+ }
+
+ public void testExample2NoIndex2() { // Figure 2 from Vigna 2012 paper, no index, test broadword selection.
+ long indexInterval = 16;
+ long[] values = new long[] {5,8,8,15,32}; // two low bits, high values 1,2,2,3,8.
+ long[] indexLongs = new long[0]; // high bits 0b 0001 0000 0101 1010
+ EliasFanoEncoder efEncVI = tstEFVI(values, indexInterval, indexLongs);
+ EliasFanoDecoder efDecVI = efEncVI.getDecoder();
+ assertEquals("initial next", 5, efDecVI.nextValue());
+ assertEquals("advance 22", 32, efDecVI.advanceToValue(22));
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/lucene4956/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Mon Oct 21 18:58:24 2013
@@ -96,8 +96,14 @@ public class TestPackedInts extends Luce
final Directory d = newDirectory();
IndexOutput out = d.createOutput("out.bin", newIOContext(random()));
- PackedInts.Writer w = PackedInts.getWriter(
- out, valueCount, nbits, random().nextFloat());
+ final float acceptableOverhead;
+ if (iter == 0) {
+ // have the first iteration go through exact nbits
+ acceptableOverhead = 0.0f;
+ } else {
+ acceptableOverhead = random().nextFloat();
+ }
+ PackedInts.Writer w = PackedInts.getWriter(out, valueCount, nbits, acceptableOverhead);
final long startFp = out.getFilePointer();
final int actualValueCount = random().nextBoolean() ? valueCount : _TestUtil.nextInt(random(), 0, valueCount);
@@ -185,8 +191,7 @@ public class TestPackedInts extends Luce
+ valueCount + " nbits=" + nbits + " for "
+ intsEnum.getClass().getSimpleName();
final int index = random().nextInt(valueCount);
- long value = intsEnum.get(index);
- assertEquals(msg, value, values[index]);
+ assertEquals(msg, values[index], intsEnum.get(index));
}
intsEnum.get(intsEnum.size() - 1);
assertEquals(fp, in.getFilePointer());
Modified: lucene/dev/branches/lucene4956/lucene/demo/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/demo/ivy.xml?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/demo/ivy.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/demo/ivy.xml Mon Oct 21 18:58:24 2013
@@ -19,7 +19,7 @@
<ivy-module version="2.0">
<info organisation="org.apache.lucene" module="core-demo"/>
<dependencies>
- <dependency org="javax.servlet" name="servlet-api" rev="2.4"/>
+ <dependency org="javax.servlet" name="servlet-api" rev="${/javax.servlet/servlet-api}"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/branches/lucene4956/lucene/demo/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/demo/src/java/overview.html?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/demo/src/java/overview.html (original)
+++ lucene/dev/branches/lucene4956/lucene/demo/src/java/overview.html Mon Oct 21 18:58:24 2013
@@ -79,9 +79,10 @@ which will contain an index of all of th
<pre>
java org.apache.lucene.demo.SearchFiles
</pre>
-You'll be prompted for a query. Type in a swear word and press the enter key.
-You'll see that the Lucene developers are very well mannered and get no
-results. Now try entering the word "string". That should return a whole bunch
+You'll be prompted for a query. Type in a gibberish or made up word (for example:
+"superca<!-- need to break up word in a way that is not visibile so it doesn't cause this ile to match a search on this word -->lifragilisticexpialidocious").
+You'll see that there are no maching results in the lucene source code.
+Now try entering the word "string". That should return a whole bunch
of documents. The results will page at every tenth result and ask you whether
you want more results.</div>
<a name="About_the_code"></a>
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42BinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42BinaryDocValues.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42BinaryDocValues.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42BinaryDocValues.java Mon Oct 21 18:58:24 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
class Facet42BinaryDocValues extends BinaryDocValues {
@@ -44,4 +45,9 @@ class Facet42BinaryDocValues extends Bin
ret.length = (int) (addresses.get(docID+1)-start);
}
+ /** Returns approximate RAM bytes used */
+ public long ramBytesUsed() {
+ return RamUsageEstimator.sizeOf(bytes) + addresses.ramBytesUsed();
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesConsumer.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesConsumer.java Mon Oct 21 18:58:24 2013
@@ -68,7 +68,9 @@ public class Facet42DocValuesConsumer ex
long totBytes = 0;
for (BytesRef v : values) {
- totBytes += v.length;
+ if (v != null) {
+ totBytes += v.length;
+ }
}
if (totBytes > Integer.MAX_VALUE) {
@@ -78,7 +80,9 @@ public class Facet42DocValuesConsumer ex
out.writeVInt((int) totBytes);
for (BytesRef v : values) {
- out.writeBytes(v.bytes, v.offset, v.length);
+ if (v != null) {
+ out.writeBytes(v.bytes, v.offset, v.length);
+ }
}
PackedInts.Writer w = PackedInts.getWriter(out, maxDoc+1, PackedInts.bitsRequired(totBytes+1), acceptableOverheadRatio);
@@ -86,7 +90,9 @@ public class Facet42DocValuesConsumer ex
int address = 0;
for(BytesRef v : values) {
w.add(address);
- address += v.length;
+ if (v != null) {
+ address += v.length;
+ }
}
w.add(address);
w.finish();
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesProducer.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesProducer.java Mon Oct 21 18:58:24 2013
@@ -31,15 +31,18 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
class Facet42DocValuesProducer extends DocValuesProducer {
private final Map<Integer,Facet42BinaryDocValues> fields = new HashMap<Integer,Facet42BinaryDocValues>();
+ private final int maxDoc;
Facet42DocValuesProducer(SegmentReadState state) throws IOException {
String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Facet42DocValuesFormat.EXTENSION);
IndexInput in = state.directory.openInput(fileName, state.context);
+ this.maxDoc = state.segmentInfo.getDocCount();
boolean success = false;
try {
CodecUtil.checkHeader(in, Facet42DocValuesFormat.CODEC,
@@ -81,6 +84,20 @@ class Facet42DocValuesProducer extends D
}
@Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ return new Bits.MatchAllBits(maxDoc); // TODO: have codec impl this?
+ }
+
+ @Override
public void close() throws IOException {
}
+
+ @Override
+ public long ramBytesUsed() {
+ long size = 0;
+ for (Facet42BinaryDocValues entry: fields.values()) {
+ size += entry.ramBytesUsed() + Integer.SIZE;
+ }
+ return size;
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java Mon Oct 21 18:58:24 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.facet.search.Fa
import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.util.Bits;
/** Uses a {@link NumericDocValues} and accumulates
* counts for provided ranges. This is dynamic (does not
@@ -86,10 +87,19 @@ public class RangeAccumulator extends Fa
if (ndv == null) {
continue; // no numeric values for this field in this reader
}
+ Bits docsWithField = hits.context.reader().getDocsWithField(ranges.field);
+
final int length = hits.bits.length();
int doc = 0;
while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) {
long v = ndv.get(doc);
+
+ // Skip missing docs:
+ if (v == 0 && docsWithField.get(doc) == false) {
+ doc++;
+ continue;
+ }
+
// TODO: if all ranges are non-overlapping, we
// should instead do a bin-search up front
// (really, a specialized case of the interval
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java Mon Oct 21 18:58:24 2013
@@ -33,7 +33,6 @@ import org.apache.lucene.facet.search.Fa
import org.apache.lucene.facet.search.FacetResultNode;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs;
-import org.apache.lucene.facet.search.TaxonomyFacetsAccumulator;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.IndexReader;
@@ -44,7 +43,7 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PriorityQueue;
-/** A {@link TaxonomyFacetsAccumulator} that uses previously
+/** A {@link FacetsAccumulator} that uses previously
* indexed {@link SortedSetDocValuesFacetFields} to perform faceting,
* without require a separate taxonomy index. Faceting is
* a bit slower (~25%), and there is added cost on every
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetFields.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetFields.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetFields.java Mon Oct 21 18:58:24 2013
@@ -31,12 +31,12 @@ import org.apache.lucene.facet.params.Fa
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.util.BytesRef;
-/** Add instances of this to your Document if you intend to
+/** Use this to index facets if you intend to
* use {@link SortedSetDocValuesAccumulator} to count facets
* at search time. Note that this only supports flat
- * facets (dimension + label). Add multiple instances of
- * this to your document, one per dimension + label, and
- * it's fine if a given dimension is multi-valued. */
+ * facets (dimension + label). Instantiate this class
+ * once, and then call {@link #addFields} to add the
+ * necessary fields to each {@link Document}. */
public class SortedSetDocValuesFacetFields extends FacetFields {
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java Mon Oct 21 18:58:24 2013
@@ -96,11 +96,7 @@ public final class SortedSetDocValuesRea
// We need this to create thread-safe MultiSortedSetDV
// per collector:
- if (reader instanceof AtomicReader) {
- topReader = (AtomicReader) reader;
- } else {
- topReader = new SlowCompositeReaderWrapper((CompositeReader) reader);
- }
+ topReader = SlowCompositeReaderWrapper.wrap(reader);
SortedSetDocValues dv = topReader.getSortedSetDocValues(field);
if (dv == null) {
throw new IllegalArgumentException("field \"" + field + "\" was not indexed with SortedSetDocValues");
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Mon Oct 21 18:58:24 2013
@@ -555,12 +555,16 @@ public class DirectoryTaxonomyWriter imp
private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private boolean returned;
+ private int val;
+ private final String word;
+
public SinglePositionTokenStream(String word) {
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- termAtt.setEmpty().append(word);
+ this.word = word;
returned = true;
}
+
/**
* Set the value we want to keep, as the position increment.
* Note that when TermPositions.nextPosition() is later used to
@@ -574,15 +578,21 @@ public class DirectoryTaxonomyWriter imp
* This change is described in Lucene's JIRA: LUCENE-1542.
*/
public void set(int val) {
- posIncrAtt.setPositionIncrement(val);
+ this.val = val;
returned = false;
}
+
@Override
public boolean incrementToken() throws IOException {
if (returned) {
return false;
}
- return returned = true;
+ clearAttributes();
+ posIncrAtt.setPositionIncrement(val);
+ termAtt.setEmpty();
+ termAtt.append(word);
+ returned = true;
+ return true;
}
}
@@ -971,12 +981,14 @@ public class DirectoryTaxonomyWriter imp
initReaderManager(); // ensure that it's initialized
refreshReaderManager();
nextID = indexWriter.maxDoc();
+ taxoArrays = null; // must nullify so that it's re-computed next time it's needed
// need to clear the cache, so that addCategory won't accidentally return
// old categories that are in the cache.
cache.clear();
cacheIsComplete = false;
shouldFillCache = true;
+ cacheMisses.set(0);
// update indexEpoch as a taxonomy replace is just like it has be recreated
++indexEpoch;
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/util/FacetsPayloadMigrationReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/util/FacetsPayloadMigrationReader.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/util/FacetsPayloadMigrationReader.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/java/org/apache/lucene/facet/util/FacetsPayloadMigrationReader.java Mon Oct 21 18:58:24 2013
@@ -40,6 +40,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
/**
@@ -223,6 +224,18 @@ public class FacetsPayloadMigrationReade
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ Term term = fieldTerms.get(field);
+ if (term == null) {
+ return super.getDocsWithField(field);
+ } else {
+ // we shouldn't return null, even if the term does not exist or has no
+ // payloads, since we already marked the field as having DocValues.
+ return new Bits.MatchAllBits(maxDoc());
+ }
+ }
+
+ @Override
public FieldInfos getFieldInfos() {
FieldInfos innerInfos = super.getFieldInfos();
ArrayList<FieldInfo> infos = new ArrayList<FieldInfo>(innerInfos.size());
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/FacetTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/FacetTestCase.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/FacetTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/FacetTestCase.java Mon Oct 21 18:58:24 2013
@@ -3,7 +3,7 @@ package org.apache.lucene.facet;
import java.util.Random;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.facet.codecs.facet42.Facet42Codec;
+import org.apache.lucene.facet.codecs.facet46.Facet46Codec;
import org.apache.lucene.facet.encoding.DGapIntEncoder;
import org.apache.lucene.facet.encoding.DGapVInt8IntEncoder;
import org.apache.lucene.facet.encoding.EightFlagsIntEncoder;
@@ -53,7 +53,7 @@ public abstract class FacetTestCase exte
public static void beforeClassFacetTestCase() throws Exception {
if (random().nextDouble() < 0.3) {
savedDefault = Codec.getDefault(); // save to restore later
- Codec.setDefault(new Facet42Codec());
+ Codec.setDefault(new Facet46Codec());
}
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/associations/AssociationsFacetRequestTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/associations/AssociationsFacetRequestTest.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/associations/AssociationsFacetRequestTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/associations/AssociationsFacetRequestTest.java Mon Oct 21 18:58:24 2013
@@ -8,9 +8,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.search.FacetResult;
-import org.apache.lucene.facet.search.FacetsAggregator;
import org.apache.lucene.facet.search.FacetsCollector;
-import org.apache.lucene.facet.search.TaxonomyFacetsAccumulator;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@@ -110,14 +108,7 @@ public class AssociationsFacetRequestTes
Query q = new MatchAllDocsQuery();
- TaxonomyFacetsAccumulator fa = new TaxonomyFacetsAccumulator(fsp, reader, taxo) {
- @Override
- public FacetsAggregator getAggregator() {
- return new SumIntAssociationFacetsAggregator();
- }
- };
-
- FacetsCollector fc = FacetsCollector.create(fa);
+ FacetsCollector fc = FacetsCollector.create(fsp, reader, taxo);
IndexSearcher searcher = newSearcher(reader);
searcher.search(q, fc);
@@ -142,14 +133,7 @@ public class AssociationsFacetRequestTes
Query q = new MatchAllDocsQuery();
- TaxonomyFacetsAccumulator fa = new TaxonomyFacetsAccumulator(fsp, reader, taxo) {
- @Override
- public FacetsAggregator getAggregator() {
- return new SumFloatAssociationFacetsAggregator();
- }
- };
-
- FacetsCollector fc = FacetsCollector.create(fa);
+ FacetsCollector fc = FacetsCollector.create(fsp, reader, taxo);
IndexSearcher searcher = newSearcher(reader);
searcher.search(q, fc);
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/old/TestScoredDocIDsUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/old/TestScoredDocIDsUtils.java?rev=1534320&r1=1534319&r2=1534320&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/old/TestScoredDocIDsUtils.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/old/TestScoredDocIDsUtils.java Mon Oct 21 18:58:24 2013
@@ -49,7 +49,7 @@ public class TestScoredDocIDsUtils exten
bits.flip(idx, idx + 1);
}
- FixedBitSet verify = new FixedBitSet(bits);
+ FixedBitSet verify = bits.clone();
ScoredDocIDs scoredDocIDs = ScoredDocIdsUtils.createScoredDocIds(bits, n);