You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by bu...@apache.org on 2010/07/22 21:34:52 UTC
svn commit: r966819 [14/20] - in /lucene/dev/branches/realtime_search: ./
lucene/ lucene/backwards/ lucene/contrib/ lucene/contrib/benchmark/conf/
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/contrib/benchmark/src/...
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestTopScoreDocCollector.java Thu Jul 22 19:34:35 2010
@@ -17,10 +17,13 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.util.Random;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
@@ -38,12 +41,12 @@ public class TestTopScoreDocCollector ex
public void testOutOfOrderCollection() throws Exception {
Directory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Random random = newRandom();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
for (int i = 0; i < 10; i++) {
writer.addDocument(new Document());
}
- writer.commit();
- writer.close();
boolean[] inOrder = new boolean[] { false, true };
String[] actualTSDCClass = new String[] {
@@ -58,7 +61,8 @@ public class TestTopScoreDocCollector ex
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.setMinimumNumberShouldMatch(1);
- IndexSearcher searcher = new IndexSearcher(dir, true);
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
for (int i = 0; i < inOrder.length; i++) {
TopDocsCollector<ScoreDoc> tdc = TopScoreDocCollector.create(3, inOrder[i]);
assertEquals("org.apache.lucene.search.TopScoreDocCollector$" + actualTSDCClass[i], tdc.getClass().getName());
@@ -71,6 +75,10 @@ public class TestTopScoreDocCollector ex
assertEquals("expected doc Id " + j + " found " + sd[j].doc, j, sd[j].doc);
}
}
+ writer.close();
+ searcher.close();
+ reader.close();
+ dir.close();
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcard.java Thu Jul 22 19:34:35 2010
@@ -19,25 +19,32 @@ package org.apache.lucene.search;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.store.RAMDirectory;
import java.io.IOException;
+import java.util.Random;
/**
* TestWildcard tests the '*' and '?' wildcard characters.
*/
public class TestWildcard
extends LuceneTestCase {
+ private Random random;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ random = newRandom();
+ }
+
public void testEquals() {
WildcardQuery wq1 = new WildcardQuery(new Term("field", "b*a"));
WildcardQuery wq2 = new WildcardQuery(new Term("field", "b*a"));
@@ -195,14 +202,13 @@ public class TestWildcard
private RAMDirectory getIndexStore(String field, String[] contents)
throws IOException {
RAMDirectory indexStore = new RAMDirectory();
- IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(
+ RandomIndexWriter writer = new RandomIndexWriter(random, indexStore, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()));
for (int i = 0; i < contents.length; ++i) {
Document doc = new Document();
doc.add(new Field(field, contents[i], Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}
- writer.optimize();
writer.close();
return indexStore;
@@ -253,7 +259,8 @@ public class TestWildcard
// prepare the index
RAMDirectory dir = new RAMDirectory();
- IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ RandomIndexWriter iw = new RandomIndexWriter(random, dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
for (int i = 0; i < docs.length; i++) {
Document doc = new Document();
doc.add(new Field(field,docs[i],Store.NO,Index.ANALYZED));
@@ -309,62 +316,4 @@ public class TestWildcard
searcher.close();
}
- @Deprecated
- private static final class OldWildcardQuery extends MultiTermQuery {
- final Term term;
-
- OldWildcardQuery(Term term) {
- this.term = term;
- }
-
- @Override
- protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
- return new WildcardTermEnum(reader, term);
- }
-
- @Override
- public String toString(String field) {
- return "OldWildcard(" + term.toString()+ ")";
- }
- }
-
- @Deprecated
- public void testDeprecatedTermEnum() throws Exception {
- RAMDirectory indexStore = getIndexStore("body", new String[]
- {"metal", "metals"});
- IndexSearcher searcher = new IndexSearcher(indexStore, true);
- Query query1 = new TermQuery(new Term("body", "metal"));
- Query query2 = new OldWildcardQuery(new Term("body", "metal*"));
- Query query3 = new OldWildcardQuery(new Term("body", "m*tal"));
- Query query4 = new OldWildcardQuery(new Term("body", "m*tal*"));
- Query query5 = new OldWildcardQuery(new Term("body", "m*tals"));
-
- BooleanQuery query6 = new BooleanQuery();
- query6.add(query5, BooleanClause.Occur.SHOULD);
-
- BooleanQuery query7 = new BooleanQuery();
- query7.add(query3, BooleanClause.Occur.SHOULD);
- query7.add(query5, BooleanClause.Occur.SHOULD);
-
- // Queries do not automatically lower-case search terms:
- Query query8 = new OldWildcardQuery(new Term("body", "M*tal*"));
-
- assertMatches(searcher, query1, 1);
- assertMatches(searcher, query2, 2);
- assertMatches(searcher, query3, 1);
- assertMatches(searcher, query4, 2);
- assertMatches(searcher, query5, 1);
- assertMatches(searcher, query6, 1);
- assertMatches(searcher, query7, 2);
- assertMatches(searcher, query8, 0);
- assertMatches(searcher, new OldWildcardQuery(new Term("body", "*tall")), 0);
- assertMatches(searcher, new OldWildcardQuery(new Term("body", "*tal")), 1);
- assertMatches(searcher, new OldWildcardQuery(new Term("body", "*tal*")), 2);
- }
-
- @Deprecated
- public void testBackwardsLayer() {
- assertTrue(new WildcardQuery(new Term("body", "metal*")).hasNewAPI);
- assertFalse(new OldWildcardQuery(new Term("body", "metal*")).hasNewAPI);
- }
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestWildcardRandom.java Thu Jul 22 19:34:35 2010
@@ -26,8 +26,11 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -40,13 +43,15 @@ import org.apache.lucene.util._TestUtil;
public class TestWildcardRandom extends LuceneTestCase {
private Searcher searcher;
private Random random;
+ private Directory dir;
@Override
protected void setUp() throws Exception {
super.setUp();
- RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new MockAnalyzer(),
- IndexWriter.MaxFieldLength.UNLIMITED);
+ random = newRandom();
+ dir = new RAMDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
Document doc = new Document();
Field field = new Field("field", "", Field.Store.NO, Field.Index.ANALYZED);
@@ -58,9 +63,9 @@ public class TestWildcardRandom extends
writer.addDocument(doc);
}
- writer.optimize();
+ IndexReader reader = writer.getReader();
+ searcher = new IndexSearcher(reader);
writer.close();
- searcher = new IndexSearcher(dir);
}
private char N() {
@@ -82,6 +87,7 @@ public class TestWildcardRandom extends
}
private void assertPatternHits(String pattern, int numHits) throws Exception {
+ // TODO: run with different rewrites
Query wq = new WildcardQuery(new Term("field", fillPattern(pattern)));
TopDocs docs = searcher.search(wq, 25);
assertEquals("Incorrect hits for pattern: " + pattern, numHits, docs.totalHits);
@@ -90,11 +96,11 @@ public class TestWildcardRandom extends
@Override
protected void tearDown() throws Exception {
searcher.close();
+ dir.close();
super.tearDown();
}
- public void testWildcards() throws Exception {
- random = newRandom(System.nanoTime());
+ public void testWildcards() throws Exception {;
for (int i = 0; i < 100*_TestUtil.getRandomMultiplier(); i++) {
assertPatternHits("NNNN", 1);
assertPatternHits("?NNN", 10);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java Thu Jul 22 19:34:35 2010
@@ -93,7 +93,6 @@ public class FunctionTestSetup extends L
public void setUp() throws Exception {
super.setUp();
// prepare a small index with just a few documents.
- super.setUp();
dir = new RAMDirectory();
anlzr = new MockAnalyzer();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, anlzr));
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Thu Jul 22 19:34:35 2010
@@ -26,9 +26,10 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Payload;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.IndexSearcher;
@@ -39,6 +40,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
@@ -47,6 +49,8 @@ import org.apache.lucene.search.Explanat
public class TestPayloadNearQuery extends LuceneTestCase {
private IndexSearcher searcher;
+ private IndexReader reader;
+ private Directory directory;
private BoostingSimilarity similarity = new BoostingSimilarity();
private byte[] payload2 = new byte[]{2};
private byte[] payload4 = new byte[]{4};
@@ -101,9 +105,10 @@ public class TestPayloadNearQuery extend
@Override
protected void setUp() throws Exception {
super.setUp();
- RAMDirectory directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new PayloadAnalyzer()).setSimilarity(similarity));
+ directory = new RAMDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer())
+ .setSimilarity(similarity));
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
@@ -112,13 +117,21 @@ public class TestPayloadNearQuery extend
doc.add(new Field("field2", txt, Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}
- writer.optimize();
+ reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(directory, true);
+ searcher = new IndexSearcher(reader);
searcher.setSimilarity(similarity);
}
+ @Override
+ protected void tearDown() throws Exception {
+ searcher.close();
+ reader.close();
+ directory.close();
+ super.tearDown();
+ }
+
public void test() throws IOException {
PayloadNearQuery query;
TopDocs hits;
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Thu Jul 22 19:34:35 2010
@@ -34,9 +34,10 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Payload;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.document.Document;
@@ -52,6 +53,7 @@ import java.io.IOException;
**/
public class TestPayloadTermQuery extends LuceneTestCase {
private IndexSearcher searcher;
+ private IndexReader reader;
private BoostingSimilarity similarity = new BoostingSimilarity();
private byte[] payloadField = new byte[]{1};
private byte[] payloadMultiField1 = new byte[]{2};
@@ -110,9 +112,9 @@ public class TestPayloadTermQuery extend
protected void setUp() throws Exception {
super.setUp();
directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new PayloadAnalyzer()).setSimilarity(
- similarity));
+ RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer())
+ .setSimilarity(similarity));
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
@@ -123,13 +125,21 @@ public class TestPayloadTermQuery extend
doc.add(new Field("multiField", English.intToEnglish(i) + " " + English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}
- writer.optimize();
+ reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(directory, true);
+ searcher = new IndexSearcher(reader);
searcher.setSimilarity(similarity);
}
+ @Override
+ protected void tearDown() throws Exception {
+ searcher.close();
+ reader.close();
+ directory.close();
+ super.tearDown();
+ }
+
public void test() throws IOException {
PayloadTermQuery query = new PayloadTermQuery(new Term("field", "seventy"),
new MaxPayloadFunction());
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java Thu Jul 22 19:34:35 2010
@@ -23,8 +23,9 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@@ -34,6 +35,7 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
@@ -52,25 +54,35 @@ import org.apache.lucene.util.LuceneTest
*/
public class TestBasics extends LuceneTestCase {
private IndexSearcher searcher;
+ private IndexReader reader;
+ private Directory directory;
@Override
protected void setUp() throws Exception {
super.setUp();
- RAMDirectory directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
+ directory = new RAMDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(newRandom(), directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}
-
+ reader = writer.getReader();
+ searcher = new IndexSearcher(reader);
writer.close();
+ }
- searcher = new IndexSearcher(directory, true);
+ @Override
+ protected void tearDown() throws Exception {
+ searcher.close();
+ reader.close();
+ directory.close();
+ super.tearDown();
}
-
+
+
public void testTerm() throws Exception {
Query query = new TermQuery(new Term("field", "seventy"));
checkHits(query, new int[]
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java Thu Jul 22 19:34:35 2010
@@ -24,13 +24,14 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
@@ -49,12 +50,15 @@ public class TestFieldMaskingSpanQuery e
}
protected IndexSearcher searcher;
+ protected Directory directory;
+ protected IndexReader reader;
@Override
protected void setUp() throws Exception {
super.setUp();
- RAMDirectory directory = new RAMDirectory();
- IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ directory = new RAMDirectory();
+ RandomIndexWriter writer= new RandomIndexWriter(newRandom(), directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
writer.addDocument(doc(new Field[] { field("id", "0")
,
@@ -109,14 +113,16 @@ public class TestFieldMaskingSpanQuery e
field("gender", "male"),
field("first", "bubba"),
field("last", "jones") }));
-
+ reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(directory, true);
+ searcher = new IndexSearcher(reader);
}
@Override
protected void tearDown() throws Exception {
searcher.close();
+ reader.close();
+ directory.close();
super.tearDown();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Thu Jul 22 19:34:35 2010
@@ -20,8 +20,9 @@ package org.apache.lucene.search.spans;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.CheckHits;
@@ -29,11 +30,14 @@ import org.apache.lucene.search.Explanat
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
public class TestNearSpansOrdered extends LuceneTestCase {
protected IndexSearcher searcher;
+ protected Directory directory;
+ protected IndexReader reader;
public static final String FIELD = "field";
public static final QueryParser qp =
@@ -42,21 +46,25 @@ public class TestNearSpansOrdered extend
@Override
protected void tearDown() throws Exception {
searcher.close();
+ reader.close();
+ directory.close();
super.tearDown();
}
@Override
protected void setUp() throws Exception {
super.setUp();
- RAMDirectory directory = new RAMDirectory();
- IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ directory = new RAMDirectory();
+ RandomIndexWriter writer= new RandomIndexWriter(newRandom(), directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(new Field(FIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
writer.addDocument(doc);
}
+ reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(directory, true);
+ searcher = new IndexSearcher(reader);
}
protected String[] docFields = {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Thu Jul 22 19:34:35 2010
@@ -33,32 +33,44 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.util.LuceneTestCase;
import java.io.IOException;
-import java.util.Collections;
public class TestSpans extends LuceneTestCase {
private IndexSearcher searcher;
+ private IndexReader reader;
+ private Directory directory;
public static final String field = "field";
@Override
protected void setUp() throws Exception {
super.setUp();
- RAMDirectory directory = new RAMDirectory();
- IndexWriter writer= new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ directory = new RAMDirectory();
+ RandomIndexWriter writer= new RandomIndexWriter(newRandom(), directory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(new Field(field, docFields[i], Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}
+ reader = writer.getReader();
writer.close();
- searcher = new IndexSearcher(directory, true);
+ searcher = new IndexSearcher(reader);
}
-
+
+ @Override
+ protected void tearDown() throws Exception {
+ searcher.close();
+ reader.close();
+ directory.close();
+ super.tearDown();
+ }
+
private String[] docFields = {
"w1 w2 w3 w4 w5",
"w1 w3 w2 w3",
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java Thu Jul 22 19:34:35 2010
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
+import java.util.Random;
import org.apache.lucene.util.LuceneTestCase;
@@ -26,8 +27,9 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
@@ -36,136 +38,145 @@ import org.apache.lucene.store.RAMDirect
/*******************************************************************************
* Tests the span query bug in Lucene. It demonstrates that SpanTermQuerys don't
* work correctly in a BooleanQuery.
- *
+ *
*/
public class TestSpansAdvanced extends LuceneTestCase {
-
- // location to the index
- protected Directory mDirectory;
-
- protected IndexSearcher searcher;
-
- // field names in the index
- private final static String FIELD_ID = "ID";
- protected final static String FIELD_TEXT = "TEXT";
-
- /**
- * Initializes the tests by adding 4 identical documents to the index.
- */
- @Override
- protected void setUp() throws Exception {
- super.setUp();
-
- // create test index
- mDirectory = new RAMDirectory();
- final IndexWriter writer = new IndexWriter(mDirectory,
- new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
- addDocument(writer, "1", "I think it should work.");
- addDocument(writer, "2", "I think it should work.");
- addDocument(writer, "3", "I think it should work.");
- addDocument(writer, "4", "I think it should work.");
- writer.close();
- searcher = new IndexSearcher(mDirectory, true);
- }
-
- @Override
- protected void tearDown() throws Exception {
- searcher.close();
- mDirectory.close();
- mDirectory = null;
- super.tearDown();
- }
-
- /**
- * Adds the document to the index.
- *
- * @param writer the Lucene index writer
- * @param id the unique id of the document
- * @param text the text of the document
- * @throws IOException
- */
- protected void addDocument(final IndexWriter writer, final String id, final String text) throws IOException {
-
- final Document document = new Document();
- document.add(new Field(FIELD_ID, id, Field.Store.YES, Field.Index.NOT_ANALYZED));
- document.add(new Field(FIELD_TEXT, text, Field.Store.YES, Field.Index.ANALYZED));
- writer.addDocument(document);
- }
-
- /**
- * Tests two span queries.
- *
- * @throws IOException
- */
- public void testBooleanQueryWithSpanQueries() throws IOException {
-
- doTestBooleanQueryWithSpanQueries(searcher,0.3884282f);
+
+ // location to the index
+ protected Directory mDirectory;
+ protected IndexReader reader;
+ protected IndexSearcher searcher;
+ protected Random random;
+
+ // field names in the index
+ private final static String FIELD_ID = "ID";
+ protected final static String FIELD_TEXT = "TEXT";
+
+ /**
+ * Initializes the tests by adding 4 identical documents to the index.
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ random = newRandom();
+ // create test index
+ mDirectory = new RAMDirectory();
+ final RandomIndexWriter writer = new RandomIndexWriter(random,
+ mDirectory, new IndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(MockTokenizer.SIMPLE, true,
+ MockTokenFilter.ENGLISH_STOPSET, true)));
+ addDocument(writer, "1", "I think it should work.");
+ addDocument(writer, "2", "I think it should work.");
+ addDocument(writer, "3", "I think it should work.");
+ addDocument(writer, "4", "I think it should work.");
+ reader = writer.getReader();
+ writer.close();
+ searcher = new IndexSearcher(reader);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ searcher.close();
+ reader.close();
+ mDirectory.close();
+ mDirectory = null;
+ super.tearDown();
+ }
+
+ /**
+ * Adds the document to the index.
+ *
+ * @param writer the Lucene index writer
+ * @param id the unique id of the document
+ * @param text the text of the document
+ * @throws IOException
+ */
+ protected void addDocument(final RandomIndexWriter writer, final String id,
+ final String text) throws IOException {
+
+ final Document document = new Document();
+ document.add(new Field(FIELD_ID, id, Field.Store.YES,
+ Field.Index.NOT_ANALYZED));
+ document.add(new Field(FIELD_TEXT, text, Field.Store.YES,
+ Field.Index.ANALYZED));
+ writer.addDocument(document);
+ }
+
+ /**
+ * Tests two span queries.
+ *
+ * @throws IOException
+ */
+ public void testBooleanQueryWithSpanQueries() throws IOException {
+
+ doTestBooleanQueryWithSpanQueries(searcher, 0.3884282f);
+ }
+
+ /**
+ * Tests two span queries.
+ *
+ * @throws IOException
+ */
+ protected void doTestBooleanQueryWithSpanQueries(IndexSearcher s,
+ final float expectedScore) throws IOException {
+
+ final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "work"));
+ final BooleanQuery query = new BooleanQuery();
+ query.add(spanQuery, BooleanClause.Occur.MUST);
+ query.add(spanQuery, BooleanClause.Occur.MUST);
+ final String[] expectedIds = new String[] {"1", "2", "3", "4"};
+ final float[] expectedScores = new float[] {expectedScore, expectedScore,
+ expectedScore, expectedScore};
+ assertHits(s, query, "two span queries", expectedIds, expectedScores);
+ }
+
+ /**
+ * Checks to see if the hits are what we expected.
+ *
+ * @param query the query to execute
+ * @param description the description of the search
+ * @param expectedIds the expected document ids of the hits
+ * @param expectedScores the expected scores of the hits
+ *
+ * @throws IOException
+ */
+ protected static void assertHits(Searcher s, Query query,
+ final String description, final String[] expectedIds,
+ final float[] expectedScores) throws IOException {
+ QueryUtils.check(query, s);
+
+ final float tolerance = 1e-5f;
+
+ // Hits hits = searcher.search(query);
+ // hits normalizes and throws things off if one score is greater than 1.0
+ TopDocs topdocs = s.search(query, null, 10000);
+
+ /*****
+ * // display the hits System.out.println(hits.length() +
+ * " hits for search: \"" + description + '\"'); for (int i = 0; i <
+ * hits.length(); i++) { System.out.println(" " + FIELD_ID + ':' +
+ * hits.doc(i).get(FIELD_ID) + " (score:" + hits.score(i) + ')'); }
+ *****/
+
+ // did we get the hits we expected
+ assertEquals(expectedIds.length, topdocs.totalHits);
+ for (int i = 0; i < topdocs.totalHits; i++) {
+ // System.out.println(i + " exp: " + expectedIds[i]);
+ // System.out.println(i + " field: " + hits.doc(i).get(FIELD_ID));
+
+ int id = topdocs.scoreDocs[i].doc;
+ float score = topdocs.scoreDocs[i].score;
+ Document doc = s.doc(id);
+ assertEquals(expectedIds[i], doc.get(FIELD_ID));
+ boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
+ if (!scoreEq) {
+ System.out.println(i + " warning, expected score: " + expectedScores[i]
+ + ", actual " + score);
+ System.out.println(s.explain(query, id));
+ }
+ assertEquals(expectedScores[i], score, tolerance);
+ assertEquals(s.explain(query, id).getValue(), score, tolerance);
}
-
- /**
- * Tests two span queries.
- *
- * @throws IOException
- */
- protected void doTestBooleanQueryWithSpanQueries(IndexSearcher s, final float expectedScore) throws IOException {
-
- final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "work"));
- final BooleanQuery query = new BooleanQuery();
- query.add(spanQuery, BooleanClause.Occur.MUST);
- query.add(spanQuery, BooleanClause.Occur.MUST);
- final String[] expectedIds = new String[] { "1", "2", "3", "4" };
- final float[] expectedScores = new float[] { expectedScore, expectedScore, expectedScore, expectedScore };
- assertHits(s, query, "two span queries", expectedIds, expectedScores);
- }
-
-
- /**
- * Checks to see if the hits are what we expected.
- *
- * @param query the query to execute
- * @param description the description of the search
- * @param expectedIds the expected document ids of the hits
- * @param expectedScores the expected scores of the hits
- *
- * @throws IOException
- */
- protected static void assertHits(Searcher s, Query query, final String description, final String[] expectedIds,
- final float[] expectedScores) throws IOException {
- QueryUtils.check(query,s);
-
- final float tolerance = 1e-5f;
-
- // Hits hits = searcher.search(query);
- // hits normalizes and throws things off if one score is greater than 1.0
- TopDocs topdocs = s.search(query,null,10000);
-
- /*****
- // display the hits
- System.out.println(hits.length() + " hits for search: \"" + description + '\"');
- for (int i = 0; i < hits.length(); i++) {
- System.out.println(" " + FIELD_ID + ':' + hits.doc(i).get(FIELD_ID) + " (score:" + hits.score(i) + ')');
- }
- *****/
-
- // did we get the hits we expected
- assertEquals(expectedIds.length, topdocs.totalHits);
- for (int i = 0; i < topdocs.totalHits; i++) {
- //System.out.println(i + " exp: " + expectedIds[i]);
- //System.out.println(i + " field: " + hits.doc(i).get(FIELD_ID));
-
- int id = topdocs.scoreDocs[i].doc;
- float score = topdocs.scoreDocs[i].score;
- Document doc = s.doc(id);
- assertEquals(expectedIds[i], doc.get(FIELD_ID));
- boolean scoreEq = Math.abs(expectedScores[i] - score) < tolerance;
- if (!scoreEq) {
- System.out.println(i + " warning, expected score: " + expectedScores[i] + ", actual " + score);
- System.out.println(s.explain(query,id));
- }
- assertEquals(expectedScores[i], score, tolerance);
- assertEquals(s.explain(query,id).getValue(), score, tolerance);
- }
- }
-
-
+ }
+
}
\ No newline at end of file
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java Thu Jul 22 19:34:35 2010
@@ -23,8 +23,8 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.*;
@@ -32,84 +32,97 @@ import org.apache.lucene.search.*;
/*******************************************************************************
* Some expanded tests to make sure my patch doesn't break other SpanTermQuery
* functionality.
- *
+ *
*/
public class TestSpansAdvanced2 extends TestSpansAdvanced {
- IndexSearcher searcher2;
- /**
- * Initializes the tests by adding documents to the index.
- */
- @Override
- protected void setUp() throws Exception {
- super.setUp();
-
- // create test index
- final IndexWriter writer = new IndexWriter(mDirectory,
- new IndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)).setOpenMode(
- OpenMode.APPEND));
- addDocument(writer, "A", "Should we, could we, would we?");
- addDocument(writer, "B", "It should. Should it?");
- addDocument(writer, "C", "It shouldn't.");
- addDocument(writer, "D", "Should we, should we, should we.");
- writer.close();
-
- // re-open the searcher since we added more docs
- searcher2 = new IndexSearcher(mDirectory, true);
- }
-
- /**
- * Verifies that the index has the correct number of documents.
- *
- * @throws Exception
- */
- public void testVerifyIndex() throws Exception {
- final IndexReader reader = IndexReader.open(mDirectory, true);
- assertEquals(8, reader.numDocs());
- reader.close();
- }
-
- /**
- * Tests a single span query that matches multiple documents.
- *
- * @throws IOException
- */
- public void testSingleSpanQuery() throws IOException {
-
- final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
- final String[] expectedIds = new String[] { "B", "D", "1", "2", "3", "4", "A" };
- final float[] expectedScores = new float[] { 0.625f, 0.45927936f, 0.35355338f, 0.35355338f, 0.35355338f,
- 0.35355338f, 0.26516503f, };
- assertHits(searcher2, spanQuery, "single span query", expectedIds, expectedScores);
- }
-
- /**
- * Tests a single span query that matches multiple documents.
- *
- * @throws IOException
- */
- public void testMultipleDifferentSpanQueries() throws IOException {
-
- final Query spanQuery1 = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
- final Query spanQuery2 = new SpanTermQuery(new Term(FIELD_TEXT, "we"));
- final BooleanQuery query = new BooleanQuery();
- query.add(spanQuery1, BooleanClause.Occur.MUST);
- query.add(spanQuery2, BooleanClause.Occur.MUST);
- final String[] expectedIds = new String[] { "D", "A" };
- // these values were pre LUCENE-413
- // final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
- final float[] expectedScores = new float[] { 1.0191123f, 0.93163157f };
- assertHits(searcher2, query, "multiple different span queries", expectedIds, expectedScores);
- }
-
- /**
- * Tests two span queries.
- *
- * @throws IOException
- */
- @Override
- public void testBooleanQueryWithSpanQueries() throws IOException {
-
- doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
- }
+ IndexSearcher searcher2;
+ IndexReader reader2;
+
+ /**
+ * Initializes the tests by adding documents to the index.
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ // create test index
+ final RandomIndexWriter writer = new RandomIndexWriter(random, mDirectory,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(
+ MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true))
+ .setOpenMode(OpenMode.APPEND));
+ addDocument(writer, "A", "Should we, could we, would we?");
+ addDocument(writer, "B", "It should. Should it?");
+ addDocument(writer, "C", "It shouldn't.");
+ addDocument(writer, "D", "Should we, should we, should we.");
+ reader2 = writer.getReader();
+ writer.close();
+
+ // re-open the searcher since we added more docs
+ searcher2 = new IndexSearcher(reader2);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ searcher2.close();
+ reader2.close();
+ super.tearDown();
+ }
+
+ /**
+ * Verifies that the index has the correct number of documents.
+ *
+ * @throws Exception
+ */
+ public void testVerifyIndex() throws Exception {
+ final IndexReader reader = IndexReader.open(mDirectory, true);
+ assertEquals(8, reader.numDocs());
+ reader.close();
+ }
+
+ /**
+ * Tests a single span query that matches multiple documents.
+ *
+ * @throws IOException
+ */
+ public void testSingleSpanQuery() throws IOException {
+
+ final Query spanQuery = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
+ final String[] expectedIds = new String[] {"B", "D", "1", "2", "3", "4",
+ "A"};
+ final float[] expectedScores = new float[] {0.625f, 0.45927936f,
+ 0.35355338f, 0.35355338f, 0.35355338f, 0.35355338f, 0.26516503f,};
+ assertHits(searcher2, spanQuery, "single span query", expectedIds,
+ expectedScores);
+ }
+
+ /**
+ * Tests a single span query that matches multiple documents.
+ *
+ * @throws IOException
+ */
+ public void testMultipleDifferentSpanQueries() throws IOException {
+
+ final Query spanQuery1 = new SpanTermQuery(new Term(FIELD_TEXT, "should"));
+ final Query spanQuery2 = new SpanTermQuery(new Term(FIELD_TEXT, "we"));
+ final BooleanQuery query = new BooleanQuery();
+ query.add(spanQuery1, BooleanClause.Occur.MUST);
+ query.add(spanQuery2, BooleanClause.Occur.MUST);
+ final String[] expectedIds = new String[] {"D", "A"};
+ // these values were pre LUCENE-413
+ // final float[] expectedScores = new float[] { 0.93163157f, 0.20698164f };
+ final float[] expectedScores = new float[] {1.0191123f, 0.93163157f};
+ assertHits(searcher2, query, "multiple different span queries",
+ expectedIds, expectedScores);
+ }
+
+ /**
+ * Tests two span queries.
+ *
+ * @throws IOException
+ */
+ @Override
+ public void testBooleanQueryWithSpanQueries() throws IOException {
+
+ doTestBooleanQueryWithSpanQueries(searcher2, 0.73500174f);
+ }
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java Thu Jul 22 19:34:35 2010
@@ -72,6 +72,9 @@ public abstract class LuceneTestCase ext
private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null;
+ /** Used to track if setUp and tearDown are called correctly from subclasses */
+ private boolean setup;
+
private static class UncaughtExceptionEntry {
public final Thread thread;
public final Throwable exception;
@@ -94,7 +97,8 @@ public abstract class LuceneTestCase ext
@Override
protected void setUp() throws Exception {
super.setUp();
-
+ assertFalse("ensure your tearDown() calls super.tearDown()!!!", setup);
+ setup = true;
savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
@@ -128,6 +132,8 @@ public abstract class LuceneTestCase ext
@Override
protected void tearDown() throws Exception {
+ assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
+ setup = false;
BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
try {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/LuceneTestCaseJ4.java Thu Jul 22 19:34:35 2010
@@ -23,6 +23,7 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.FieldCache.CacheEntry;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -33,6 +34,7 @@ import java.io.File;
import java.io.PrintStream;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Hashtable;
import java.util.Iterator;
import java.util.Random;
import java.util.ArrayList;
@@ -106,6 +108,9 @@ public class LuceneTestCaseJ4 {
private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null;
+ /** Used to track if setUp and tearDown are called correctly from subclasses */
+ private boolean setup;
+
private static class UncaughtExceptionEntry {
public final Thread thread;
public final Throwable exception;
@@ -155,6 +160,8 @@ public class LuceneTestCaseJ4 {
@Before
public void setUp() throws Exception {
+ Assert.assertFalse("ensure your tearDown() calls super.tearDown()!!!", setup);
+ setup = true;
savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
@@ -191,6 +198,8 @@ public class LuceneTestCaseJ4 {
@After
public void tearDown() throws Exception {
+ Assert.assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
+ setup = false;
BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
try {
@@ -330,6 +339,31 @@ public class LuceneTestCaseJ4 {
return new Random(seed);
}
+ private static final Map<Class<? extends LuceneTestCaseJ4>,Long> staticSeeds =
+ Collections.synchronizedMap(new WeakHashMap<Class<? extends LuceneTestCaseJ4>,Long>());
+
+ /**
+ * Returns a {@link Random} instance for generating random numbers from a beforeclass
+ * annotated method.
+ * The random seed is logged during test execution and printed to System.out on any failure
+ * for reproducing the test using {@link #newStaticRandom(Class, long)} with the recorded seed
+ * .
+ */
+ public static Random newStaticRandom(Class<? extends LuceneTestCaseJ4> clazz) {
+ return newStaticRandom(clazz, seedRnd.nextLong());
+ }
+
+ /**
+ * Returns a {@link Random} instance for generating random numbers from a beforeclass
+ * annotated method.
+ * If an error occurs in the test that is not reproducible, you can use this method to
+ * initialize the number generator with the seed that was printed out during the failing test.
+ */
+ public static Random newStaticRandom(Class<? extends LuceneTestCaseJ4> clazz, long seed) {
+ staticSeeds.put(clazz, Long.valueOf(seed));
+ return new Random(seed);
+ }
+
public String getName() {
return this.name;
}
@@ -348,6 +382,11 @@ public class LuceneTestCaseJ4 {
// We get here from InterceptTestCaseEvents on the 'failed' event....
public void reportAdditionalFailureInfo() {
+ Long staticSeed = staticSeeds.get(getClass());
+ if (staticSeed != null) {
+ System.out.println("NOTE: random static seed of testclass '" + getName() + "' was: " + staticSeed);
+ }
+
if (seed != null) {
System.out.println("NOTE: random seed of testcase '" + getName() + "' was: " + seed);
}
Propchange: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jul 22 19:34:35 2010
@@ -1,4 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137,949730
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137,949730,957490,960490,961612
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:953476-966816
/lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java:824912-931101
/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:909334,948516
/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-924731,924781,925176-925462
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/TestNumericUtils.java Thu Jul 22 19:34:35 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
+import java.util.Random;
public class TestNumericUtils extends LuceneTestCase {
@@ -30,7 +31,7 @@ public class TestNumericUtils extends Lu
NumericUtils.longToPrefixCoded(l, 0, act);
if (last!=null) {
// test if smaller
- assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 );
+ assertTrue("actual bigger than last (BytesRef)", last.compareTo(act) < 0 );
assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 );
}
// test is back and forward conversion works
@@ -48,7 +49,7 @@ public class TestNumericUtils extends Lu
NumericUtils.intToPrefixCoded(i, 0, act);
if (last!=null) {
// test if smaller
- assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 );
+ assertTrue("actual bigger than last (BytesRef)", last.compareTo(act) < 0 );
assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 );
}
// test is back and forward conversion works
@@ -84,7 +85,7 @@ public class TestNumericUtils extends Lu
// check sort order (prefixVals should be ascending)
for (int i=1; i<prefixVals.length; i++) {
- assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 );
+ assertTrue( "check sort order", prefixVals[i-1].compareTo(prefixVals[i]) < 0 );
}
// check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
@@ -124,7 +125,7 @@ public class TestNumericUtils extends Lu
// check sort order (prefixVals should be ascending)
for (int i=1; i<prefixVals.length; i++) {
- assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 );
+ assertTrue( "check sort order", prefixVals[i-1].compareTo(prefixVals[i]) < 0 );
}
// check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
@@ -179,23 +180,30 @@ public class TestNumericUtils extends Lu
// INFO: Tests for trieCodeLong()/trieCodeInt() not needed because implicitely tested by range filter tests
- /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
- protected void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
- final boolean useBitSet, final Iterator<Long> neededBounds
+ /** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */
+ private void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
+ final boolean useBitSet, final Iterable<Long> expectedBounds, final Iterable<Integer> expectedShifts
) throws Exception {
final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
-
+ final Iterator<Long> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
+ final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
+
NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
@Override
public void addRange(long min, long max, int shift) {
assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
if (useBitSet) for (long l=min; l<=max; l++) {
assertFalse("ranges should not overlap", bits.getAndSet(l-lower) );
+ // extra exit condition to prevent overflow on MAX_VALUE
+ if (l == max) break;
}
+ if (neededBounds == null || neededShifts == null)
+ return;
// make unsigned longs for easier display and understanding
min ^= 0x8000000000000000L;
max ^= 0x8000000000000000L;
- //System.out.println("Long.valueOf(0x"+Long.toHexString(min>>>shift)+"L),Long.valueOf(0x"+Long.toHexString(max>>>shift)+"L),");
+ //System.out.println("0x"+Long.toHexString(min>>>shift)+"L,0x"+Long.toHexString(max>>>shift)+"L)/*shift="+shift+"*/,");
+ assertEquals( "shift", neededShifts.next().intValue(), shift);
assertEquals( "inner min bound", neededBounds.next().longValue(), min>>>shift);
assertEquals( "inner max bound", neededBounds.next().longValue(), max>>>shift);
}
@@ -208,64 +216,218 @@ public class TestNumericUtils extends Lu
}
}
+ /** LUCENE-2541: NumericRangeQuery errors with endpoints near long min and max values */
+ public void testLongExtremeValues() throws Exception {
+ // upper end extremes
+ assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 1, true, Arrays.asList(
+ 0xffffffffffffffffL,0xffffffffffffffffL
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 2, true, Arrays.asList(
+ 0xffffffffffffffffL,0xffffffffffffffffL
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 4, true, Arrays.asList(
+ 0xffffffffffffffffL,0xffffffffffffffffL
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 6, true, Arrays.asList(
+ 0xffffffffffffffffL,0xffffffffffffffffL
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 8, true, Arrays.asList(
+ 0xffffffffffffffffL,0xffffffffffffffffL
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MAX_VALUE, Long.MAX_VALUE, 64, true, Arrays.asList(
+ 0xffffffffffffffffL,0xffffffffffffffffL
+ ), Arrays.asList(
+ 0
+ ));
+
+ assertLongRangeSplit(Long.MAX_VALUE-0xfL, Long.MAX_VALUE, 4, true, Arrays.asList(
+ 0xfffffffffffffffL,0xfffffffffffffffL
+ ), Arrays.asList(
+ 4
+ ));
+ assertLongRangeSplit(Long.MAX_VALUE-0x10L, Long.MAX_VALUE, 4, true, Arrays.asList(
+ 0xffffffffffffffefL,0xffffffffffffffefL,
+ 0xfffffffffffffffL,0xfffffffffffffffL
+ ), Arrays.asList(
+ 0, 4
+ ));
+
+ // lower end extremes
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 1, true, Arrays.asList(
+ 0x0000000000000000L,0x0000000000000000L
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 2, true, Arrays.asList(
+ 0x0000000000000000L,0x0000000000000000L
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 4, true, Arrays.asList(
+ 0x0000000000000000L,0x0000000000000000L
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 6, true, Arrays.asList(
+ 0x0000000000000000L,0x0000000000000000L
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 8, true, Arrays.asList(
+ 0x0000000000000000L,0x0000000000000000L
+ ), Arrays.asList(
+ 0
+ ));
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE, 64, true, Arrays.asList(
+ 0x0000000000000000L,0x0000000000000000L
+ ), Arrays.asList(
+ 0
+ ));
+
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE+0xfL, 4, true, Arrays.asList(
+ 0x000000000000000L,0x000000000000000L
+ ), Arrays.asList(
+ 4
+ ));
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MIN_VALUE+0x10L, 4, true, Arrays.asList(
+ 0x0000000000000010L,0x0000000000000010L,
+ 0x000000000000000L,0x000000000000000L
+ ), Arrays.asList(
+ 0, 4
+ ));
+ }
+
+ public void testRandomSplit() throws Exception {
+ final Random random = newRandom();
+ long num = 100L * _TestUtil.getRandomMultiplier();
+ for (long i=0; i < num; i++) {
+ executeOneRandomSplit(random);
+ }
+ }
+
+ private void executeOneRandomSplit(final Random random) throws Exception {
+ long lower = randomLong(random);
+ long len = (long) random.nextInt(16384*1024); // not too large bitsets, else OOME!
+ while (lower + len < lower) { // overflow
+ lower >>= 1;
+ }
+ assertLongRangeSplit(lower, lower + len, random.nextInt(64) + 1, true, null, null);
+ }
+
+ private long randomLong(final Random random) {
+ long val;
+ switch(random.nextInt(4)) {
+ case 0:
+ val = 1L << (random.nextInt(63)); // patterns like 0x000000100000 (-1 yields patterns like 0x0000fff)
+ break;
+ case 1:
+ val = -1L << (random.nextInt(63)); // patterns like 0xfffff00000
+ break;
+ default:
+ val = random.nextLong();
+ }
+
+ val += random.nextInt(5)-2;
+
+ if (random.nextBoolean()) {
+ if (random.nextBoolean()) val += random.nextInt(100)-50;
+ if (random.nextBoolean()) val = ~val;
+ if (random.nextBoolean()) val = val<<1;
+ if (random.nextBoolean()) val = val>>>1;
+ }
+
+ return val;
+ }
+
public void testSplitLongRange() throws Exception {
// a hard-coded "standard" range
- assertLongRangeSplit(-5000L, 9500L, 4, true, Arrays.asList(new Long[]{
- Long.valueOf(0x7fffffffffffec78L),Long.valueOf(0x7fffffffffffec7fL),
- Long.valueOf(0x8000000000002510L),Long.valueOf(0x800000000000251cL),
- Long.valueOf(0x7fffffffffffec8L), Long.valueOf(0x7fffffffffffecfL),
- Long.valueOf(0x800000000000250L), Long.valueOf(0x800000000000250L),
- Long.valueOf(0x7fffffffffffedL), Long.valueOf(0x7fffffffffffefL),
- Long.valueOf(0x80000000000020L), Long.valueOf(0x80000000000024L),
- Long.valueOf(0x7ffffffffffffL), Long.valueOf(0x8000000000001L)
- }).iterator());
+ assertLongRangeSplit(-5000L, 9500L, 4, true, Arrays.asList(
+ 0x7fffffffffffec78L,0x7fffffffffffec7fL,
+ 0x8000000000002510L,0x800000000000251cL,
+ 0x7fffffffffffec8L, 0x7fffffffffffecfL,
+ 0x800000000000250L, 0x800000000000250L,
+ 0x7fffffffffffedL, 0x7fffffffffffefL,
+ 0x80000000000020L, 0x80000000000024L,
+ 0x7ffffffffffffL, 0x8000000000001L
+ ), Arrays.asList(
+ 0, 0,
+ 4, 4,
+ 8, 8,
+ 12
+ ));
// the same with no range splitting
- assertLongRangeSplit(-5000L, 9500L, 64, true, Arrays.asList(new Long[]{
- Long.valueOf(0x7fffffffffffec78L),Long.valueOf(0x800000000000251cL)
- }).iterator());
+ assertLongRangeSplit(-5000L, 9500L, 64, true, Arrays.asList(
+ 0x7fffffffffffec78L,0x800000000000251cL
+ ), Arrays.asList(
+ 0
+ ));
// this tests optimized range splitting, if one of the inner bounds
// is also the bound of the next lower precision, it should be used completely
- assertLongRangeSplit(0L, 1024L+63L, 4, true, Arrays.asList(new Long[]{
- Long.valueOf(0x800000000000040L), Long.valueOf(0x800000000000043L),
- Long.valueOf(0x80000000000000L), Long.valueOf(0x80000000000003L)
- }).iterator());
+ assertLongRangeSplit(0L, 1024L+63L, 4, true, Arrays.asList(
+ 0x800000000000040L, 0x800000000000043L,
+ 0x80000000000000L, 0x80000000000003L
+ ), Arrays.asList(
+ 4, 8
+ ));
// the full long range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
- assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 8, false, Arrays.asList(new Long[]{
- Long.valueOf(0x00L),Long.valueOf(0xffL)
- }).iterator());
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 8, false, Arrays.asList(
+ 0x00L,0xffL
+ ), Arrays.asList(
+ 56
+ ));
// the same with precisionStep=4
- assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 4, false, Arrays.asList(new Long[]{
- Long.valueOf(0x0L),Long.valueOf(0xfL)
- }).iterator());
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 4, false, Arrays.asList(
+ 0x0L,0xfL
+ ), Arrays.asList(
+ 60
+ ));
// the same with precisionStep=2
- assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 2, false, Arrays.asList(new Long[]{
- Long.valueOf(0x0L),Long.valueOf(0x3L)
- }).iterator());
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 2, false, Arrays.asList(
+ 0x0L,0x3L
+ ), Arrays.asList(
+ 62
+ ));
// the same with precisionStep=1
- assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 1, false, Arrays.asList(new Long[]{
- Long.valueOf(0x0L),Long.valueOf(0x1L)
- }).iterator());
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 1, false, Arrays.asList(
+ 0x0L,0x1L
+ ), Arrays.asList(
+ 63
+ ));
// a inverse range should produce no sub-ranges
- assertLongRangeSplit(9500L, -5000L, 4, false, Collections. <Long> emptyList().iterator());
+ assertLongRangeSplit(9500L, -5000L, 4, false, Collections.<Long>emptyList(), Collections.<Integer>emptyList());
// a 0-length range should reproduce the range itsself
- assertLongRangeSplit(9500L, 9500L, 4, false, Arrays.asList(new Long[]{
- Long.valueOf(0x800000000000251cL),Long.valueOf(0x800000000000251cL)
- }).iterator());
+ assertLongRangeSplit(9500L, 9500L, 4, false, Arrays.asList(
+ 0x800000000000251cL,0x800000000000251cL
+ ), Arrays.asList(
+ 0
+ ));
}
- /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
- protected void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
- final boolean useBitSet, final Iterator<Integer> neededBounds
+ /** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */
+ private void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
+ final boolean useBitSet, final Iterable<Integer> expectedBounds, final Iterable<Integer> expectedShifts
) throws Exception {
final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+ final Iterator<Integer> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
+ final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
@Override
@@ -273,11 +435,16 @@ public class TestNumericUtils extends Lu
assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
if (useBitSet) for (int i=min; i<=max; i++) {
assertFalse("ranges should not overlap", bits.getAndSet(i-lower) );
+ // extra exit condition to prevent overflow on MAX_VALUE
+ if (i == max) break;
}
+ if (neededBounds == null)
+ return;
// make unsigned ints for easier display and understanding
min ^= 0x80000000;
max ^= 0x80000000;
- //System.out.println("Integer.valueOf(0x"+Integer.toHexString(min>>>shift)+"),Integer.valueOf(0x"+Integer.toHexString(max>>>shift)+"),");
+ //System.out.println("0x"+Integer.toHexString(min>>>shift)+",0x"+Integer.toHexString(max>>>shift)+")/*shift="+shift+"*/,");
+ assertEquals( "shift", neededShifts.next().intValue(), shift);
assertEquals( "inner min bound", neededBounds.next().intValue(), min>>>shift);
assertEquals( "inner max bound", neededBounds.next().intValue(), max>>>shift);
}
@@ -292,55 +459,74 @@ public class TestNumericUtils extends Lu
public void testSplitIntRange() throws Exception {
// a hard-coded "standard" range
- assertIntRangeSplit(-5000, 9500, 4, true, Arrays.asList(new Integer[]{
- Integer.valueOf(0x7fffec78),Integer.valueOf(0x7fffec7f),
- Integer.valueOf(0x80002510),Integer.valueOf(0x8000251c),
- Integer.valueOf(0x7fffec8), Integer.valueOf(0x7fffecf),
- Integer.valueOf(0x8000250), Integer.valueOf(0x8000250),
- Integer.valueOf(0x7fffed), Integer.valueOf(0x7fffef),
- Integer.valueOf(0x800020), Integer.valueOf(0x800024),
- Integer.valueOf(0x7ffff), Integer.valueOf(0x80001)
- }).iterator());
+ assertIntRangeSplit(-5000, 9500, 4, true, Arrays.asList(
+ 0x7fffec78,0x7fffec7f,
+ 0x80002510,0x8000251c,
+ 0x7fffec8, 0x7fffecf,
+ 0x8000250, 0x8000250,
+ 0x7fffed, 0x7fffef,
+ 0x800020, 0x800024,
+ 0x7ffff, 0x80001
+ ), Arrays.asList(
+ 0, 0,
+ 4, 4,
+ 8, 8,
+ 12
+ ));
// the same with no range splitting
- assertIntRangeSplit(-5000, 9500, 32, true, Arrays.asList(new Integer[]{
- Integer.valueOf(0x7fffec78),Integer.valueOf(0x8000251c)
- }).iterator());
+ assertIntRangeSplit(-5000, 9500, 32, true, Arrays.asList(
+ 0x7fffec78,0x8000251c
+ ), Arrays.asList(
+ 0
+ ));
// this tests optimized range splitting, if one of the inner bounds
// is also the bound of the next lower precision, it should be used completely
- assertIntRangeSplit(0, 1024+63, 4, true, Arrays.asList(new Integer[]{
- Integer.valueOf(0x8000040), Integer.valueOf(0x8000043),
- Integer.valueOf(0x800000), Integer.valueOf(0x800003)
- }).iterator());
+ assertIntRangeSplit(0, 1024+63, 4, true, Arrays.asList(
+ 0x8000040, 0x8000043,
+ 0x800000, 0x800003
+ ), Arrays.asList(
+ 4, 8
+ ));
// the full int range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
- assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 8, false, Arrays.asList(new Integer[]{
- Integer.valueOf(0x00),Integer.valueOf(0xff)
- }).iterator());
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 8, false, Arrays.asList(
+ 0x00,0xff
+ ), Arrays.asList(
+ 24
+ ));
// the same with precisionStep=4
- assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 4, false, Arrays.asList(new Integer[]{
- Integer.valueOf(0x0),Integer.valueOf(0xf)
- }).iterator());
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 4, false, Arrays.asList(
+ 0x0,0xf
+ ), Arrays.asList(
+ 28
+ ));
// the same with precisionStep=2
- assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 2, false, Arrays.asList(new Integer[]{
- Integer.valueOf(0x0),Integer.valueOf(0x3)
- }).iterator());
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 2, false, Arrays.asList(
+ 0x0,0x3
+ ), Arrays.asList(
+ 30
+ ));
// the same with precisionStep=1
- assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 1, false, Arrays.asList(new Integer[]{
- Integer.valueOf(0x0),Integer.valueOf(0x1)
- }).iterator());
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 1, false, Arrays.asList(
+ 0x0,0x1
+ ), Arrays.asList(
+ 31
+ ));
// a inverse range should produce no sub-ranges
- assertIntRangeSplit(9500, -5000, 4, false, Collections. <Integer> emptyList().iterator());
+ assertIntRangeSplit(9500, -5000, 4, false, Collections.<Integer>emptyList(), Collections.<Integer>emptyList());
// a 0-length range should reproduce the range itsself
- assertIntRangeSplit(9500, 9500, 4, false, Arrays.asList(new Integer[]{
- Integer.valueOf(0x8000251c),Integer.valueOf(0x8000251c)
- }).iterator());
+ assertIntRangeSplit(9500, 9500, 4, false, Arrays.asList(
+ 0x8000251c,0x8000251c
+ ), Arrays.asList(
+ 0
+ ));
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/AutomatonTestUtil.java Thu Jul 22 19:34:35 2010
@@ -17,8 +17,17 @@ package org.apache.lucene.util.automaton
* limitations under the License.
*/
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import java.util.Random;
+import java.util.Set;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
@@ -65,4 +74,143 @@ public class AutomatonTestUtil {
}
return new String(buffer, 0, end);
}
+
+ // picks a random int code point that this transition
+ // accepts, avoiding the surrogates range since they are
+ // "defined" in UTF32. Don't call this on a transition
+ // that only accepts UTF16 surrogate values!!
+ private static int getRandomCodePoint(final Random r, final Transition t) {
+ return t.min+r.nextInt(t.max-t.min+1);
+ }
+
+ public static class RandomAcceptedStrings {
+
+ private final Map<Transition,Boolean> leadsToAccept;
+ private final Automaton a;
+
+ private static class ArrivingTransition {
+ final State from;
+ final Transition t;
+ public ArrivingTransition(State from, Transition t) {
+ this.from = from;
+ this.t = t;
+ }
+ }
+
+ public RandomAcceptedStrings(Automaton a) {
+ this.a = a;
+ if (a.isSingleton()) {
+ leadsToAccept = null;
+ return;
+ }
+
+ // must use IdentityHashmap because two Transitions w/
+ // different start nodes can be considered the same
+ leadsToAccept = new IdentityHashMap<Transition,Boolean>();
+ final Map<State,List<ArrivingTransition>> allArriving = new HashMap<State,List<ArrivingTransition>>();
+
+ final LinkedList<State> q = new LinkedList<State>();
+ final Set<State> seen = new HashSet<State>();
+
+ // reverse map the transitions, so we can quickly look
+ // up all arriving transitions to a given state
+ for(State s: a.getNumberedStates()) {
+ for(int i=0;i<s.numTransitions;i++) {
+ final Transition t = s.transitionsArray[i];
+ List<ArrivingTransition> tl = allArriving.get(t.to);
+ if (tl == null) {
+ tl = new ArrayList<ArrivingTransition>();
+ allArriving.put(t.to, tl);
+ }
+ tl.add(new ArrivingTransition(s, t));
+ }
+ if (s.accept) {
+ q.add(s);
+ seen.add(s);
+ }
+ }
+
+ // Breadth-first search, from accept states,
+ // backwards:
+ while(!q.isEmpty()) {
+ final State s = q.removeFirst();
+ List<ArrivingTransition> arriving = allArriving.get(s);
+ if (arriving != null) {
+ for(ArrivingTransition at : arriving) {
+ final State from = at.from;
+ if (!seen.contains(from)) {
+ q.add(from);
+ seen.add(from);
+ leadsToAccept.put(at.t, Boolean.TRUE);
+ }
+ }
+ }
+ }
+ }
+
+ public int[] getRandomAcceptedString(Random r) {
+
+ final List<Integer> soFar = new ArrayList<Integer>();
+ if (a.isSingleton()) {
+ // accepts only one
+ final String s = a.singleton;
+
+ int charUpto = 0;
+ while(charUpto < s.length()) {
+ final int cp = s.codePointAt(charUpto);
+ charUpto += Character.charCount(cp);
+ soFar.add(cp);
+ }
+ } else {
+
+ State s = a.initial;
+
+ while(true) {
+
+ if (s.accept) {
+ if (s.numTransitions == 0) {
+ // stop now
+ break;
+ } else {
+ if (r.nextBoolean()) {
+ break;
+ }
+ }
+ }
+
+ if (s.numTransitions == 0) {
+ throw new RuntimeException("this automaton has dead states");
+ }
+
+ boolean cheat = r.nextBoolean();
+
+ final Transition t;
+ if (cheat) {
+ // pick a transition that we know is the fastest
+ // path to an accept state
+ List<Transition> toAccept = new ArrayList<Transition>();
+ for(int i=0;i<s.numTransitions;i++) {
+ final Transition t0 = s.transitionsArray[i];
+ if (leadsToAccept.containsKey(t0)) {
+ toAccept.add(t0);
+ }
+ }
+ if (toAccept.size() == 0) {
+ // this is OK -- it means we jumped into a cycle
+ t = s.transitionsArray[r.nextInt(s.numTransitions)];
+ } else {
+ t = toAccept.get(r.nextInt(toAccept.size()));
+ }
+ } else {
+ t = s.transitionsArray[r.nextInt(s.numTransitions)];
+ }
+
+ soFar.add(getRandomCodePoint(r, t));
+ s = t.to;
+ }
+ }
+
+ return ArrayUtil.toIntArray(soFar);
+ }
+ }
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestBasicOperations.java Thu Jul 22 19:34:35 2010
@@ -95,7 +95,7 @@ public class TestBasicOperations extends
final Automaton a = re.toAutomaton();
assertFalse(BasicOperations.isEmpty(a));
- final BasicOperations.RandomAcceptedStrings rx = new BasicOperations.RandomAcceptedStrings(a);
+ final AutomatonTestUtil.RandomAcceptedStrings rx = new AutomatonTestUtil.RandomAcceptedStrings(a);
for(int j=0;j<ITER2;j++) {
int[] acc = null;
try {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java Thu Jul 22 19:34:35 2010
@@ -35,7 +35,7 @@ public class TestDeterminizeLexicon exte
private List<String> terms = new ArrayList<String>();
private Random random;
- public void testLexicon() {
+ public void testLexicon() throws Exception {
random = newRandom();
for (int i = 0; i < 3*_TestUtil.getRandomMultiplier(); i++) {
automata.clear();
@@ -49,7 +49,7 @@ public class TestDeterminizeLexicon exte
}
}
- public void assertLexicon() {
+ public void assertLexicon() throws Exception {
Collections.shuffle(automata, random);
final Automaton lex = BasicOperations.union(automata);
lex.determinize();
@@ -59,8 +59,8 @@ public class TestDeterminizeLexicon exte
}
final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex);
for (String s : terms) {
- BytesRef termByte = new BytesRef(s);
- assertTrue(lexByte.run(termByte.bytes, 0, termByte.length));
+ byte bytes[] = s.getBytes("UTF-8");
+ assertTrue(lexByte.run(bytes, 0, bytes.length));
}
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java Thu Jul 22 19:34:35 2010
@@ -173,7 +173,7 @@ public class TestUTF32ToUTF8 extends Luc
private void assertAutomaton(Automaton automaton) throws Exception {
CharacterRunAutomaton cra = new CharacterRunAutomaton(automaton);
ByteRunAutomaton bra = new ByteRunAutomaton(automaton);
- final BasicOperations.RandomAcceptedStrings ras = new BasicOperations.RandomAcceptedStrings(automaton);
+ final AutomatonTestUtil.RandomAcceptedStrings ras = new AutomatonTestUtil.RandomAcceptedStrings(automaton);
for (int i = 0; i < 1000*_TestUtil.getRandomMultiplier(); i++) {
final String string;
@@ -193,8 +193,8 @@ public class TestUTF32ToUTF8 extends Luc
throw e;
}
}
- BytesRef bytesRef = new BytesRef(string);
- assertEquals(cra.run(string), bra.run(bytesRef.bytes, 0, bytesRef.length));
+ byte bytes[] = string.getBytes("UTF-8");
+ assertEquals(cra.run(string), bra.run(bytes, 0, bytes.length));
}
}
}
Modified: lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt?rev=966819&r1=966818&r2=966819&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/modules/analysis/CHANGES.txt Thu Jul 22 19:34:35 2010
@@ -27,11 +27,14 @@ New Features
with text contained in the required words (inverse of StopFilter).
- o.a.l.analysis.miscellaneous.HyphenatedWordsFilter: A TokenFilter that puts
hyphenated words broken into two lines back together.
+ - o.a.l.analysis.miscellaneous.CapitalizationFilter: A TokenFilter that applies
+ capitalization rules to tokens.
- o.a.l.analysis.pattern: Package for pattern-based analysis, containing a
CharFilter, Tokenizer, and Tokenfilter for transforming text with regexes.
- o.a.l.analysis.synonym.SynonymFilter: A synonym filter that supports multi-word
synonyms.
- (... in progress)
+ - o.a.l.analysis.phonetic: Package for phonetic search, containing various
+ phonetic encoders such as Double Metaphone.
* LUCENE-2413: Consolidated all Lucene analyzers into common.
- o.a.l.analysis.KeywordAnalyzer -> o.a.l.analysis.core.KeywordAnalyzer
@@ -60,7 +63,6 @@ New Features
- o.a.l.analysis.ReusableAnalyzerBase -> o.a.l.analysis.util.ReusableAnalyzerBase
- o.a.l.analysis.StopwordAnalyzerBase -> o.a.l.analysis.util.StopwordAnalyzerBase
- o.a.l.analysis.WordListLoader -> o.a.l.analysis.util.WordListLoader
- ... (in progress)
Build