You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2011/03/15 22:35:35 UTC
svn commit: r1081952 [9/17] - in /lucene/dev/branches/bulkpostings: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/
dev-tools/idea/lucene/ dev-tools/idea/lucene/contrib/ant/
dev-tools/idea/lucene/contrib/demo/ dev-tools/idea/luce...
Modified: lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java Tue Mar 15 21:35:17 2011
@@ -38,6 +38,18 @@ import org.apache.lucene.util._TestUtil;
/**
* This is a Directory Wrapper that adds methods
* intended to be used only by unit tests.
+ * It also adds a number of features useful for testing:
+ * <ul>
+ * <li> Instances created by {@link LuceneTestCase#newDirectory()} are tracked
+ * to ensure they are closed by the test.
+ * <li> When a MockDirectoryWrapper is closed, it will throw an exception if
+ * it has any open files against it (with a stacktrace indicating where
+ * they were opened from).
+ * <li> When a MockDirectoryWrapper is closed, it runs CheckIndex to test if
+ * the index was corrupted.
+ * <li> MockDirectoryWrapper simulates some "features" of Windows, such as
+ * refusing to write/delete to open files.
+ * </ul>
*/
public class MockDirectoryWrapper extends Directory {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java Tue Mar 15 21:35:17 2011
@@ -32,9 +32,9 @@ import java.util.Random;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-// Minimal port of contrib/benchmark's LneDocSource +
-// DocMaker, so tests can enum docs from a line file created
-// by contrib/benchmark's WriteLineDoc task
+/** Minimal port of contrib/benchmark's LneDocSource +
+ * DocMaker, so tests can enum docs from a line file created
+ * by contrib/benchmark's WriteLineDoc task */
public class LineFileDocs implements Closeable {
private BufferedReader reader;
@@ -42,8 +42,8 @@ public class LineFileDocs implements Clo
private final AtomicInteger id = new AtomicInteger();
private final String path;
- // If forever is true, we rewind the file at EOF (repeat
- // the docs over and over)
+ /** If forever is true, we rewind the file at EOF (repeat
+ * the docs over and over) */
public LineFileDocs(Random random, String path) throws IOException {
this.path = path;
open(random);
@@ -138,7 +138,7 @@ public class LineFileDocs implements Clo
private final ThreadLocal<DocState> threadDocs = new ThreadLocal<DocState>();
- // Document instance is re-used per-thread
+ /** Note: Document instance is re-used per-thread */
public Document nextDoc() throws IOException {
String line;
synchronized(this) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Tue Mar 15 21:35:17 2011
@@ -83,9 +83,9 @@ import org.junit.runners.model.Initializ
* <code>super.tearDown()</code>
* </p>
*
- * @After - replaces setup
- * @Before - replaces teardown
- * @Test - any public method with this annotation is a test case, regardless
+ * <code>@After</code> - replaces setup
+ * <code>@Before</code> - replaces teardown
+ * <code>@Test</code> - any public method with this annotation is a test case, regardless
* of its name
* <p>
* <p>
@@ -586,7 +586,8 @@ public abstract class LuceneTestCase ext
// try to stop the thread:
t.setUncaughtExceptionHandler(null);
Thread.setDefaultUncaughtExceptionHandler(null);
- t.interrupt();
+ if (!t.getName().startsWith("SyncThread")) // avoid zookeeper jre crash
+ t.interrupt();
try {
t.join(THREAD_STOP_GRACE_MSEC);
} catch (InterruptedException e) { e.printStackTrace(); }
@@ -612,7 +613,7 @@ public abstract class LuceneTestCase ext
* directly in the same scope as the IndexReader.
* </p>
*
- * @see FieldCacheSanityChecker
+ * @see org.apache.lucene.util.FieldCacheSanityChecker
*/
protected void assertSaneFieldCaches(final String msg) {
final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
@@ -708,7 +709,7 @@ public abstract class LuceneTestCase ext
}
/**
- * Convinience method for logging an iterator.
+ * Convenience method for logging an iterator.
*
* @param label String logged before/after the items in the iterator
* @param iter Each next() is toString()ed and logged on it's own line. If iter is null this is logged differnetly then an empty iterator.
@@ -728,7 +729,7 @@ public abstract class LuceneTestCase ext
}
/**
- * Convinience method for logging an array. Wraps the array in an iterator and delegates
+ * Convenience method for logging an array. Wraps the array in an iterator and delegates
*
* @see #dumpIterator(String,Iterator,PrintStream)
*/
@@ -743,6 +744,7 @@ public abstract class LuceneTestCase ext
return newIndexWriterConfig(random, v, a);
}
+ /** create a new index writer config with random defaults using the specified random */
public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) {
IndexWriterConfig c = new IndexWriterConfig(v, a);
if (r.nextBoolean()) {
@@ -835,6 +837,10 @@ public abstract class LuceneTestCase ext
return newDirectory(random);
}
+ /**
+ * Returns a new Directory instance, using the specified random.
+ * See {@link #newDirectory()} for more information.
+ */
public static MockDirectoryWrapper newDirectory(Random r) throws IOException {
Directory impl = newDirectoryImpl(r, TEST_DIRECTORY);
MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl);
@@ -889,6 +895,11 @@ public abstract class LuceneTestCase ext
}
}
+ /**
+ * Returns a new Directory instance, using the specified random
+ * with contents copied from the provided directory. See
+ * {@link #newDirectory()} for more information.
+ */
public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException {
Directory impl = newDirectoryImpl(r, TEST_DIRECTORY);
for (String file : d.listAll()) {
@@ -899,26 +910,45 @@ public abstract class LuceneTestCase ext
return dir;
}
+ /** Returns a new field instance.
+ * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(String name, String value, Index index) {
return newField(random, name, value, index);
}
+ /** Returns a new field instance.
+ * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(String name, String value, Store store, Index index) {
return newField(random, name, value, store, index);
}
+ /**
+ * Returns a new Field instance. Use this when the test does not
+ * care about some specific field settings (most tests)
+ * <ul>
+ * <li>If the store value is set to Store.NO, sometimes the field will be randomly stored.
+ * <li>More term vector data than you ask for might be indexed, for example if you choose YES
+ * it might index term vectors with offsets too.
+ * </ul>
+ */
public static Field newField(String name, String value, Store store, Index index, TermVector tv) {
return newField(random, name, value, store, index, tv);
}
+ /** Returns a new field instance, using the specified random.
+ * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(Random random, String name, String value, Index index) {
return newField(random, name, value, Store.NO, index);
}
+ /** Returns a new field instance, using the specified random.
+ * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(Random random, String name, String value, Store store, Index index) {
return newField(random, name, value, store, index, TermVector.NO);
}
+ /** Returns a new field instance, using the specified random.
+ * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(Random random, String name, String value, Store store, Index index, TermVector tv) {
if (!index.isIndexed())
return new Field(name, value, store, index);
@@ -1029,7 +1059,8 @@ public abstract class LuceneTestCase ext
}
}
- /** create a new searcher over the reader */
+ /** create a new searcher over the reader.
+ * This searcher might randomly use threads. */
public static IndexSearcher newSearcher(IndexReader r) throws IOException {
if (random.nextBoolean()) {
return new IndexSearcher(r);
@@ -1243,7 +1274,7 @@ public abstract class LuceneTestCase ext
}
@Override
- public String toString() {
+ public synchronized String toString() {
return "RandomCodecProvider: " + previousMappings.toString();
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Tue Mar 15 21:35:17 2011
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.lang.reflect.Method;
import java.util.Enumeration;
import java.util.Random;
import java.util.Map;
@@ -51,6 +52,9 @@ public class _TestUtil {
return new File(LuceneTestCase.TEMP_DIR, desc + "." + new Random().nextLong());
}
+ /**
+ * Deletes a directory and everything underneath it.
+ */
public static void rmDir(File dir) throws IOException {
if (dir.exists()) {
for (File f : dir.listFiles()) {
@@ -154,6 +158,9 @@ public class _TestUtil {
return randomUnicodeString(r, 20);
}
+ /**
+ * Returns a random string up to a certain length.
+ */
public static String randomUnicodeString(Random r, int maxLength) {
final int end = r.nextInt(maxLength);
if (end == 0) {
@@ -279,8 +286,8 @@ public class _TestUtil {
}
}
- // just tries to configure things to keep the open file
- // count lowish
+ /** just tries to configure things to keep the open file
+ * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
// keep number of open files lowish
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
@@ -305,4 +312,17 @@ public class _TestUtil {
});
Assert.assertEquals("Reflection does not produce same map", reflectedValues, map);
}
+
+ public static void keepFullyDeletedSegments(IndexWriter w) {
+ try {
+ // Carefully invoke what is a package-private (test
+ // only, internal) method on IndexWriter:
+ Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
+ m.setAccessible(true);
+ m.invoke(w);
+ } catch (Exception e) {
+ // Should not happen?
+ throw new RuntimeException(e);
+ }
+ }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java Tue Mar 15 21:35:17 2011
@@ -80,9 +80,9 @@ public class AutomatonTestUtil {
return new String(buffer, 0, end);
}
- // picks a random int code point, avoiding surrogates;
- // throws IllegalArgumentException if this transition only
- // accepts surrogates
+ /** picks a random int code point, avoiding surrogates;
+ * throws IllegalArgumentException if this transition only
+ * accepts surrogates */
private static int getRandomCodePoint(final Random r, final Transition t) {
final int code;
if (t.max < UnicodeUtil.UNI_SUR_HIGH_START ||
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Tue Mar 15 21:35:17 2011
@@ -599,139 +599,13 @@ public class TestExternalCodecs extends
}
}
- public static class MyCodecs extends CodecProvider {
- MyCodecs() {
- Codec ram = new RAMOnlyCodec();
- register(ram);
- setDefaultFieldCodec(ram.name);
- }
- }
-
- // copied from PulsingCodec, just changing the terms
- // comparator
- private static class PulsingReverseTermsCodec extends Codec {
-
- public PulsingReverseTermsCodec() {
- name = "PulsingReverseTerms";
- }
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docsWriter = new StandardPostingsWriter(state);
-
- // Terms that have <= freqCutoff number of docs are
- // "pulsed" (inlined):
- final int freqCutoff = 1;
- PostingsWriterBase pulsingWriter = new PulsingPostingsWriterImpl(freqCutoff, docsWriter);
-
- // Terms dict index
- TermsIndexWriterBase indexWriter;
- boolean success = false;
- try {
- indexWriter = new FixedGapTermsIndexWriter(state) {
- // We sort in reverse unicode order, so, we must
- // disable the suffix-stripping opto that
- // FixedGapTermsIndexWriter does by default!
- @Override
- protected int indexedTermPrefixLength(BytesRef priorTerm, BytesRef indexedTerm) {
- return indexedTerm.length;
- }
- };
- success = true;
- } finally {
- if (!success) {
- pulsingWriter.close();
- }
- }
-
- // Terms dict
- success = false;
- try {
- FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, pulsingWriter, reverseUnicodeComparator);
- success = true;
- return ret;
- } finally {
- if (!success) {
- try {
- pulsingWriter.close();
- } finally {
- indexWriter.close();
- }
- }
- }
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
-
- PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId);
- PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader);
-
- // Terms dict index reader
- TermsIndexReaderBase indexReader;
-
- boolean success = false;
- try {
- indexReader = new FixedGapTermsIndexReader(state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- state.termsIndexDivisor,
- reverseUnicodeComparator,
- state.codecId);
- success = true;
- } finally {
- if (!success) {
- pulsingReader.close();
- }
- }
-
- // Terms dict reader
- success = false;
- try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- pulsingReader,
- state.readBufferSize,
- reverseUnicodeComparator,
- StandardCodec.TERMS_CACHE_SIZE,
- state.codecId);
- success = true;
- return ret;
- } finally {
- if (!success) {
- try {
- pulsingReader.close();
- } finally {
- indexReader.close();
- }
- }
- }
- }
-
- @Override
- public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set<String> files) throws IOException {
- StandardPostingsReader.files(dir, segmentInfo, codecId, files);
- BlockTermsReader.files(dir, segmentInfo, codecId, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
- }
-
- @Override
- public void getExtensions(Set<String> extensions) {
- StandardCodec.getStandardExtensions(extensions);
- }
- }
-
-
// tests storing "id" and "field2" fields as pulsing codec,
// whose term sort is backwards unicode code point, and
// storing "field1" as a custom entirely-in-RAM codec
public void testPerFieldCodec() throws Exception {
- CodecProvider provider = new MyCodecs();
- Codec pulsing = new PulsingReverseTermsCodec();
- provider.register(pulsing);
-
+ CodecProvider provider = new CoreCodecProvider();
+ provider.register(new RAMOnlyCodec());
+ provider.setDefaultFieldCodec("RamOnly");
final int NUM_DOCS = 173;
MockDirectoryWrapper dir = newDirectory();
@@ -748,11 +622,11 @@ public class TestExternalCodecs extends
doc.add(newField("field1", "this field uses the standard codec as the test", Field.Store.NO, Field.Index.ANALYZED));
// uses pulsing codec:
Field field2 = newField("field2", "this field uses the pulsing codec as the test", Field.Store.NO, Field.Index.ANALYZED);
- provider.setFieldCodec(field2.name(), pulsing.name);
+ provider.setFieldCodec(field2.name(), "Pulsing");
doc.add(field2);
Field idField = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
- provider.setFieldCodec(idField.name(), pulsing.name);
+ provider.setFieldCodec(idField.name(), "Pulsing");
doc.add(idField);
for(int i=0;i<NUM_DOCS;i++) {
@@ -762,16 +636,13 @@ public class TestExternalCodecs extends
w.commit();
}
}
+ if (VERBOSE) {
+ System.out.println("TEST: now delete id=77");
+ }
w.deleteDocuments(new Term("id", "77"));
IndexReader r = IndexReader.open(w, true);
IndexReader[] subs = r.getSequentialSubReaders();
- // test each segment
- for(int i=0;i<subs.length;i++) {
- testTermsOrder(subs[i]);
- }
- // test each multi-reader
- testTermsOrder(r);
assertEquals(NUM_DOCS-1, r.numDocs());
IndexSearcher s = newSearcher(r);
@@ -792,7 +663,6 @@ public class TestExternalCodecs extends
assertEquals(0, s.search(new TermQuery(new Term("id", "77")), 1).totalHits);
assertEquals(0, s.search(new TermQuery(new Term("id", "44")), 1).totalHits);
- testTermsOrder(r);
r.close();
s.close();
@@ -800,25 +670,4 @@ public class TestExternalCodecs extends
dir.close();
}
-
- private void testTermsOrder(IndexReader r) throws Exception {
-
- // Verify sort order matches what my comparator said:
- BytesRef lastBytesRef = null;
- TermsEnum terms = MultiFields.getFields(r).terms("id").iterator();
- //System.out.println("id terms:");
- while(true) {
- BytesRef t = terms.next();
- if (t == null) {
- break;
- }
- //System.out.println(" " + t);
- if (lastBytesRef == null) {
- lastBytesRef = new BytesRef(t);
- } else {
- assertTrue("terms in wrong order last=" + lastBytesRef.utf8ToString() + " current=" + t.utf8ToString(), reverseUnicodeComparator.compare(lastBytesRef, t) < 0);
- lastBytesRef.copy(t);
- }
- }
- }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Mar 15 21:35:17 2011
@@ -37,6 +37,7 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
@@ -515,8 +516,8 @@ public class TestBackwardsCompatibility
assertEquals("didn't delete the right number of documents", 1, delCount);
// Set one norm so we get a .s0 file:
- SimilarityProvider sim = new DefaultSimilarity();
- reader.setNorm(21, "content", sim.get("content").encodeNormValue(1.5f));
+ Similarity sim = new DefaultSimilarity();
+ reader.setNorm(21, "content", sim.encodeNormValue(1.5f));
reader.close();
// The numbering of fields can vary depending on which
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Tue Mar 15 21:35:17 2011
@@ -68,7 +68,7 @@ public class TestIndexFileDeleter extend
Term searchTerm = new Term("id", "7");
int delCount = reader.deleteDocuments(searchTerm);
assertEquals("didn't delete the right number of documents", 1, delCount);
- Similarity sim = new DefaultSimilarity().get("content");
+ Similarity sim = new DefaultSimilarity();
// Set one norm so we get a .s0 file:
reader.setNorm(21, "content", sim.encodeNormValue(1.5f));
reader.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Mar 15 21:35:17 2011
@@ -465,7 +465,7 @@ public class TestIndexReader extends Luc
// expected
}
- Similarity sim = new DefaultSimilarity().get("aaa");
+ Similarity sim = new DefaultSimilarity();
try {
reader.setNorm(5, "aaa", sim.encodeNormValue(2.0f));
fail("setNorm after close failed to throw IOException");
@@ -506,7 +506,7 @@ public class TestIndexReader extends Luc
// expected
}
- Similarity sim = new DefaultSimilarity().get("aaa");
+ Similarity sim = new DefaultSimilarity();
try {
reader.setNorm(5, "aaa", sim.encodeNormValue(2.0f));
fail("setNorm should have hit LockObtainFailedException");
@@ -538,7 +538,7 @@ public class TestIndexReader extends Luc
// now open reader & set norm for doc 0
IndexReader reader = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity().get("content");
+ Similarity sim = new DefaultSimilarity();
reader.setNorm(0, "content", sim.encodeNormValue(2.0f));
// we should be holding the write lock now:
@@ -583,7 +583,7 @@ public class TestIndexReader extends Luc
addDoc(writer, searchTerm.text());
writer.close();
- Similarity sim = new DefaultSimilarity().get("content");
+ Similarity sim = new DefaultSimilarity();
// now open reader & set norm for doc 0 (writes to
// _0_1.s0)
reader = IndexReader.open(dir, false);
@@ -969,7 +969,7 @@ public class TestIndexReader extends Luc
dir.setMaxSizeInBytes(thisDiskFree);
dir.setRandomIOExceptionRate(rate);
- Similarity sim = new DefaultSimilarity().get("content");
+ Similarity sim = new DefaultSimilarity();
try {
if (0 == x) {
int docId = 12;
@@ -1124,7 +1124,7 @@ public class TestIndexReader extends Luc
}
reader = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity().get("content");
+ Similarity sim = new DefaultSimilarity();
try {
reader.setNorm(1, "content", sim.encodeNormValue(2.0f));
fail("did not hit exception when calling setNorm on an invalid doc number");
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java Tue Mar 15 21:35:17 2011
@@ -273,7 +273,7 @@ public class TestIndexReaderClone extend
* @throws Exception
*/
private void performDefaultTests(IndexReader r1) throws Exception {
- Similarity sim = new DefaultSimilarity().get("field1");
+ Similarity sim = new DefaultSimilarity();
float norm1 = sim.decodeNormValue(MultiNorms.norms(r1, "field1")[4]);
IndexReader pr1Clone = (IndexReader) r1.clone();
@@ -329,7 +329,7 @@ public class TestIndexReaderClone extend
TestIndexReaderReopen.createIndex(random, dir1, false);
SegmentReader origSegmentReader = getOnlySegmentReader(IndexReader.open(dir1, false));
origSegmentReader.deleteDocument(1);
- Similarity sim = new DefaultSimilarity().get("field1");
+ Similarity sim = new DefaultSimilarity();
origSegmentReader.setNorm(4, "field1", sim.encodeNormValue(0.5f));
SegmentReader clonedSegmentReader = (SegmentReader) origSegmentReader
@@ -429,7 +429,7 @@ public class TestIndexReaderClone extend
final Directory dir1 = newDirectory();
TestIndexReaderReopen.createIndex(random, dir1, false);
IndexReader orig = IndexReader.open(dir1, false);
- Similarity sim = new DefaultSimilarity().get("field1");
+ Similarity sim = new DefaultSimilarity();
orig.setNorm(1, "field1", sim.encodeNormValue(17.0f));
final byte encoded = sim.encodeNormValue(17.0f);
assertEquals(encoded, MultiNorms.norms(orig, "field1")[1]);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Tue Mar 15 21:35:17 2011
@@ -211,7 +211,7 @@ public class TestIndexReaderCloneNorms e
IndexReader reader4C = (IndexReader) reader3C.clone();
SegmentReader segmentReader4C = getOnlySegmentReader(reader4C);
assertEquals(4, reader3CCNorm.bytesRef().get());
- Similarity sim = new DefaultSimilarity().get("field1");
+ Similarity sim = new DefaultSimilarity();
reader4C.setNorm(5, "field1", sim.encodeNormValue(0.33f));
// generate a cannot update exception in reader1
@@ -272,7 +272,7 @@ public class TestIndexReaderCloneNorms e
// System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm);
modifiedNorms.set(i, Float.valueOf(newNorm));
modifiedNorms.set(k, Float.valueOf(origNorm));
- Similarity sim = new DefaultSimilarity().get("f" + 1);
+ Similarity sim = new DefaultSimilarity();
ir.setNorm(i, "f" + 1, sim.encodeNormValue(newNorm));
ir.setNorm(k, "f" + 1, sim.encodeNormValue(origNorm));
// System.out.println("setNorm i: "+i);
@@ -294,7 +294,7 @@ public class TestIndexReaderCloneNorms e
assertEquals("number of norms mismatches", numDocNorms, b.length);
ArrayList<Float> storedNorms = (i == 1 ? modifiedNorms : norms);
for (int j = 0; j < b.length; j++) {
- Similarity sim = new DefaultSimilarity().get(field);
+ Similarity sim = new DefaultSimilarity();
float norm = sim.decodeNormValue(b[j]);
float norm1 = storedNorms.get(j).floatValue();
assertEquals("stored norm value of " + field + " for doc " + j + " is "
@@ -334,7 +334,7 @@ public class TestIndexReaderCloneNorms e
// return unique norm values that are unchanged by encoding/decoding
private float nextNorm(String fname) {
float norm = lastNorm + normDelta;
- Similarity sim = new DefaultSimilarity().get(fname);
+ Similarity sim = new DefaultSimilarity();
do {
float norm1 = sim.decodeNormValue(
sim.encodeNormValue(norm));
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Tue Mar 15 21:35:17 2011
@@ -617,9 +617,9 @@ public class TestIndexReaderReopen exten
IndexReader reader2 = reader1.reopen();
modifier = IndexReader.open(dir1, false);
- SimilarityProvider sim = new DefaultSimilarity();
- modifier.setNorm(1, "field1", sim.get("field1").encodeNormValue(50f));
- modifier.setNorm(1, "field2", sim.get("field2").encodeNormValue(50f));
+ Similarity sim = new DefaultSimilarity();
+ modifier.setNorm(1, "field1", sim.encodeNormValue(50f));
+ modifier.setNorm(1, "field2", sim.encodeNormValue(50f));
modifier.close();
IndexReader reader3 = reader2.reopen();
@@ -712,7 +712,7 @@ public class TestIndexReaderReopen exten
protected void modifyIndex(int i) throws IOException {
if (i % 3 == 0) {
IndexReader modifier = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity().get("field1");
+ Similarity sim = new DefaultSimilarity();
modifier.setNorm(i, "field1", sim.encodeNormValue(50f));
modifier.close();
} else if (i % 3 == 1) {
@@ -993,10 +993,10 @@ public class TestIndexReaderReopen exten
}
case 1: {
IndexReader reader = IndexReader.open(dir, false);
- SimilarityProvider sim = new DefaultSimilarity();
- reader.setNorm(4, "field1", sim.get("field1").encodeNormValue(123f));
- reader.setNorm(44, "field2", sim.get("field2").encodeNormValue(222f));
- reader.setNorm(44, "field4", sim.get("field4").encodeNormValue(22f));
+ Similarity sim = new DefaultSimilarity();
+ reader.setNorm(4, "field1", sim.encodeNormValue(123f));
+ reader.setNorm(44, "field2", sim.encodeNormValue(222f));
+ reader.setNorm(44, "field4", sim.encodeNormValue(22f));
reader.close();
break;
}
@@ -1017,9 +1017,9 @@ public class TestIndexReaderReopen exten
}
case 4: {
IndexReader reader = IndexReader.open(dir, false);
- SimilarityProvider sim = new DefaultSimilarity();
- reader.setNorm(5, "field1", sim.get("field1").encodeNormValue(123f));
- reader.setNorm(55, "field2", sim.get("field2").encodeNormValue(222f));
+ Similarity sim = new DefaultSimilarity();
+ reader.setNorm(5, "field1", sim.encodeNormValue(123f));
+ reader.setNorm(55, "field2", sim.encodeNormValue(222f));
reader.close();
break;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Mar 15 21:35:17 2011
@@ -21,54 +21,58 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
+import java.io.Reader;
import java.io.StringReader;
-import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Set;
-import java.util.HashSet;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.Collections;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
-import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.ThreadInterruptedException;
-import org.apache.lucene.util.BytesRef;
public class TestIndexWriter extends LuceneTestCase {
@@ -2550,7 +2554,7 @@ public class TestIndexWriter extends Luc
final Random r = random;
Directory dir = newDirectory();
- FlushCountingIndexWriter w = new FlushCountingIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMaxBufferedDeleteTerms(-1));
+ FlushCountingIndexWriter w = new FlushCountingIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(-1).setMaxBufferedDeleteTerms(-1));
w.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));
@@ -2577,7 +2581,7 @@ public class TestIndexWriter extends Luc
count++;
}
}
- assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 1500);
+ assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 3000);
}
w.close();
dir.close();
@@ -2785,6 +2789,9 @@ public class TestIndexWriter extends Luc
final String id = ""+i;
idField.setValue(id);
docs.put(id, doc);
+ if (VERBOSE) {
+ System.out.println("TEST: add doc id=" + id);
+ }
for(int field: fieldIDs) {
final String s;
@@ -2803,7 +2810,7 @@ public class TestIndexWriter extends Luc
if (rand.nextInt(5) == 3 && i > 0) {
final String delID = ""+rand.nextInt(i);
if (VERBOSE) {
- System.out.println("TEST: delete doc " + delID);
+ System.out.println("TEST: delete doc id=" + delID);
}
w.deleteDocuments(new Term("id", delID));
docs.remove(delID);
@@ -2826,6 +2833,9 @@ public class TestIndexWriter extends Luc
for(int iter=0;iter<1000*RANDOM_MULTIPLIER;iter++) {
String testID = idsList[rand.nextInt(idsList.length)];
+ if (VERBOSE) {
+ System.out.println("TEST: test id=" + testID);
+ }
TopDocs hits = s.search(new TermQuery(new Term("id", testID)), 1);
assertEquals(1, hits.totalHits);
Document doc = r.document(hits.scoreDocs[0].doc);
@@ -3036,4 +3046,123 @@ public class TestIndexWriter extends Luc
w.close();
dir.close();
}
+
+ private static class StringSplitAnalyzer extends Analyzer {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new StringSplitTokenizer(reader);
+ }
+ }
+
+ private static class StringSplitTokenizer extends Tokenizer {
+ private final String[] tokens;
+ private int upto = 0;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ public StringSplitTokenizer(Reader r) {
+ try {
+ final StringBuilder b = new StringBuilder();
+ final char[] buffer = new char[1024];
+ int n;
+ while((n = r.read(buffer)) != -1) {
+ b.append(buffer, 0, n);
+ }
+ tokens = b.toString().split(" ");
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ clearAttributes();
+ if (upto < tokens.length) {
+ termAtt.setEmpty();
+ termAtt.append(tokens[upto]);
+ upto++;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Make sure we skip wicked long terms.
+ */
+ public void testWickedLongTerm() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random, dir, new StringSplitAnalyzer());
+
+ char[] chars = new char[DocumentsWriter.MAX_TERM_LENGTH_UTF8];
+ Arrays.fill(chars, 'x');
+ Document doc = new Document();
+ final String bigTerm = new String(chars);
+ final BytesRef bigTermBytesRef = new BytesRef(bigTerm);
+
+ // This contents produces a too-long term:
+ String contents = "abc xyz x" + bigTerm + " another term";
+ doc.add(new Field("content", contents, Field.Store.NO, Field.Index.ANALYZED));
+ w.addDocument(doc);
+
+ // Make sure we can add another normal document
+ doc = new Document();
+ doc.add(new Field("content", "abc bbb ccc", Field.Store.NO, Field.Index.ANALYZED));
+ w.addDocument(doc);
+
+ IndexReader reader = w.getReader();
+ w.close();
+
+ // Make sure all terms < max size were indexed
+ assertEquals(2, reader.docFreq(new Term("content", "abc")));
+ assertEquals(1, reader.docFreq(new Term("content", "bbb")));
+ assertEquals(1, reader.docFreq(new Term("content", "term")));
+ assertEquals(1, reader.docFreq(new Term("content", "another")));
+
+ // Make sure position is still incremented when
+ // massive term is skipped:
+ DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"));
+ assertEquals(0, tps.nextDoc());
+ assertEquals(1, tps.freq());
+ assertEquals(3, tps.nextPosition());
+
+ // Make sure the doc that has the massive term is in
+ // the index:
+ assertEquals("document with wicked long term should is not in the index!", 2, reader.numDocs());
+
+ reader.close();
+ dir.close();
+ dir = newDirectory();
+
+ // Make sure we can add a document with exactly the
+ // maximum length term, and search on that term:
+ doc = new Document();
+ Field contentField = new Field("content", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
+ doc.add(contentField);
+
+ w = new RandomIndexWriter(random, dir);
+
+ contentField.setValue("other");
+ w.addDocument(doc);
+
+ contentField.setValue("term");
+ w.addDocument(doc);
+
+ contentField.setValue(bigTerm);
+ w.addDocument(doc);
+
+ contentField.setValue("zzz");
+ w.addDocument(doc);
+
+ reader = w.getReader();
+ w.close();
+ assertEquals(1, reader.docFreq(new Term("content", bigTerm)));
+
+ FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(reader, "content", random.nextBoolean());
+ assertEquals(5, dti.numOrd()); // +1 for null ord
+ assertEquals(4, dti.size());
+ assertEquals(bigTermBytesRef, dti.lookup(3, new BytesRef()));
+ reader.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Mar 15 21:35:17 2011
@@ -81,7 +81,7 @@ public class TestIndexWriterDelete exten
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(2)
.setMaxBufferedDeleteTerms(2));
-
+ modifier.setInfoStream(VERBOSE ? System.out : null);
int id = 0;
int value = 100;
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Tue Mar 15 21:35:17 2011
@@ -464,11 +464,11 @@ public class TestIndexWriterOnDiskFull e
setReaderPooling(true).
setMergePolicy(newLogMergePolicy(2))
);
+ _TestUtil.keepFullyDeletedSegments(w);
Document doc = new Document();
doc.add(newField("f", "doctor who", Field.Store.YES, Field.Index.ANALYZED));
w.addDocument(doc);
-
w.commit();
w.deleteDocuments(new Term("f", "who"));
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue Mar 15 21:35:17 2011
@@ -124,7 +124,11 @@ public class TestIndexWriterReader exten
} else {
iwc.setMergePolicy(NoMergePolicy.COMPOUND_FILES);
}
+ if (VERBOSE) {
+ System.out.println("TEST: make index");
+ }
IndexWriter writer = new IndexWriter(dir1, iwc);
+ writer.setInfoStream(VERBOSE ? System.out : null);
// create the index
createIndexNoClose(!optimize, "index1", writer);
@@ -146,6 +150,9 @@ public class TestIndexWriterReader exten
IndexReader r2 = writer.getReader();
assertTrue(r2.isCurrent());
assertEquals(0, count(new Term("id", id10), r2));
+ if (VERBOSE) {
+ System.out.println("TEST: verify id");
+ }
assertEquals(1, count(new Term("id", Integer.toString(8000)), r2));
r1.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIsCurrent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIsCurrent.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIsCurrent.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIsCurrent.java Tue Mar 15 21:35:17 2011
@@ -68,7 +68,7 @@ public class TestIsCurrent extends Lucen
// assert index has a document and reader is up2date
assertEquals("One document should be in the index", 1, writer.numDocs());
- assertTrue("Document added, reader should be stale ", reader.isCurrent());
+ assertTrue("One document added, reader should be current", reader.isCurrent());
// remove document
Term idTerm = new Term("UUID", "1");
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMultiFields.java Tue Mar 15 21:35:17 2011
@@ -32,6 +32,7 @@ public class TestMultiFields extends Luc
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
+ _TestUtil.keepFullyDeletedSegments(w);
Map<BytesRef,List<Integer>> docs = new HashMap<BytesRef,List<Integer>>();
Set<Integer> deleted = new HashSet<Integer>();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java Tue Mar 15 21:35:17 2011
@@ -36,6 +36,7 @@ public class TestNRTReaderWithThreads ex
setMaxBufferedDocs(10).
setMergePolicy(newLogMergePolicy(false,2))
);
+ writer.setInfoStream(VERBOSE ? System.out : null);
IndexReader reader = writer.getReader(); // start pooling readers
reader.close();
RunThread[] indexThreads = new RunThread[4];
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java Tue Mar 15 21:35:17 2011
@@ -170,7 +170,7 @@ public class TestNorms extends LuceneTes
//System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm);
modifiedNorms.set(i, Float.valueOf(newNorm));
modifiedNorms.set(k, Float.valueOf(origNorm));
- Similarity sim = new DefaultSimilarity().get("f"+1);
+ Similarity sim = new DefaultSimilarity();
ir.setNorm(i, "f"+1, sim.encodeNormValue(newNorm));
ir.setNorm(k, "f"+1, sim.encodeNormValue(origNorm));
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestParallelReader.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestParallelReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestParallelReader.java Tue Mar 15 21:35:17 2011
@@ -149,8 +149,8 @@ public class TestParallelReader extends
assertTrue(pr.isCurrent());
IndexReader modifier = IndexReader.open(dir1, false);
- SimilarityProvider sim = new DefaultSimilarity();
- modifier.setNorm(0, "f1", sim.get("f1").encodeNormValue(100f));
+ Similarity sim = new DefaultSimilarity();
+ modifier.setNorm(0, "f1", sim.encodeNormValue(100f));
modifier.close();
// one of the two IndexReaders which ParallelReader is using
@@ -158,7 +158,7 @@ public class TestParallelReader extends
assertFalse(pr.isCurrent());
modifier = IndexReader.open(dir2, false);
- modifier.setNorm(0, "f3", sim.get("f3").encodeNormValue(100f));
+ modifier.setNorm(0, "f3", sim.encodeNormValue(100f));
modifier.close();
// now both are not current anymore
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java Tue Mar 15 21:35:17 2011
@@ -71,7 +71,7 @@ public class TestThreadedOptimize extend
}
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4);
- //writer.setInfoStream(System.out);
+ writer.setInfoStream(VERBOSE ? System.out : null);
Thread[] threads = new Thread[NUM_THREADS];
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java Tue Mar 15 21:35:17 2011
@@ -577,50 +577,6 @@ public class TestQueryParser extends Luc
assertQueryEquals("[\\* TO \"*\"]",null,"[\\* TO \\*]");
}
- public void testFarsiRangeCollating() throws Exception {
- Directory ramDir = newDirectory();
- IndexWriter iw = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
- Document doc = new Document();
- doc.add(newField("content","\u0633\u0627\u0628",
- Field.Store.YES, Field.Index.NOT_ANALYZED));
- iw.addDocument(doc);
- iw.close();
- IndexSearcher is = new IndexSearcher(ramDir, true);
-
- QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "content", new MockAnalyzer(MockTokenizer.WHITESPACE, false));
-
- // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
- // RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
- // characters properly.
- Collator c = Collator.getInstance(new Locale("ar"));
- qp.setRangeCollator(c);
-
- // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
- // orders the U+0698 character before the U+0633 character, so the single
- // index Term below should NOT be returned by a ConstantScoreRangeQuery
- // with a Farsi Collator (or an Arabic one for the case when Farsi is not
- // supported).
-
- // Test ConstantScoreRangeQuery
- qp.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
- ScoreDoc[] result = is.search(qp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs;
- assertEquals("The index Term should not be included.", 0, result.length);
-
- result = is.search(qp.parse("[ \u0633 TO \u0638 ]"), null, 1000).scoreDocs;
- assertEquals("The index Term should be included.", 1, result.length);
-
- // Test TermRangeQuery
- qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
- result = is.search(qp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs;
- assertEquals("The index Term should not be included.", 0, result.length);
-
- result = is.search(qp.parse("[ \u0633 TO \u0638 ]"), null, 1000).scoreDocs;
- assertEquals("The index Term should be included.", 1, result.length);
-
- is.close();
- ramDir.close();
- }
-
private String escapeDateString(String s) {
if (s.indexOf(" ") > -1) {
return "\"" + s + "\"";
@@ -1260,4 +1216,41 @@ public class TestQueryParser extends Luc
Query unexpanded = new TermQuery(new Term("field", "dogs"));
assertEquals(unexpanded, smart.parse("\"dogs\""));
}
+
+ /**
+ * Mock collation analyzer: indexes terms as "collated" + term
+ */
+ private class MockCollationFilter extends TokenFilter {
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ protected MockCollationFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAtt.toString();
+ termAtt.setEmpty().append("collated").append(term);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ }
+ private class MockCollationAnalyzer extends Analyzer {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new MockCollationFilter(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ }
+ }
+
+ public void testCollatedRange() throws Exception {
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockCollationAnalyzer());
+ qp.setAnalyzeRangeTerms(true);
+ Query expected = TermRangeQuery.newStringRange("field", "collatedabc", "collateddef", true, true);
+ Query actual = qp.parse("[abc TO def]");
+ assertEquals(expected, actual);
+ }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestAutomatonQuery.java Tue Mar 15 21:35:17 2011
@@ -79,6 +79,9 @@ public class TestAutomatonQuery extends
}
private int automatonQueryNrHits(AutomatonQuery query) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: run aq=" + query);
+ }
return searcher.search(query, 5).totalHits;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java Tue Mar 15 21:35:17 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestCachingSpanFilter extends LuceneTestCase {
@@ -73,7 +74,9 @@ public class TestCachingSpanFilter exten
docs = searcher.search(constantScore, 1);
assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
- // now delete the doc, refresh the reader, and see that it's not there
+ // now delete the doc, refresh the reader, and see that
+ // it's not there
+ _TestUtil.keepFullyDeletedSegments(writer.w);
writer.deleteDocuments(new Term("id", "1"));
reader = refreshReader(reader);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Tue Mar 15 21:35:17 2011
@@ -22,8 +22,8 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SlowMultiReaderWrapper;
@@ -32,6 +32,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetDISI;
+import org.apache.lucene.util._TestUtil;
public class TestCachingWrapperFilter extends LuceneTestCase {
@@ -196,6 +197,7 @@ public class TestCachingWrapperFilter ex
assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
// now delete the doc, refresh the reader, and see that it's not there
+ _TestUtil.keepFullyDeletedSegments(writer.w);
writer.deleteDocuments(new Term("id", "1"));
reader = refreshReader(reader);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java Tue Mar 15 21:35:17 2011
@@ -35,7 +35,7 @@ public class TestConstantScoreQuery exte
public void testCSQ() throws Exception {
final Query q1 = new ConstantScoreQuery(new TermQuery(new Term("a", "b")));
final Query q2 = new ConstantScoreQuery(new TermQuery(new Term("a", "c")));
- final Query q3 = new ConstantScoreQuery(new TermRangeFilter("a", "b", "c", true, true));
+ final Query q3 = new ConstantScoreQuery(TermRangeFilter.newStringRange("a", "b", "c", true, true));
QueryUtils.check(q1);
QueryUtils.check(q2);
QueryUtils.checkEqual(q1,q1);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDateFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDateFilter.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDateFilter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDateFilter.java Tue Mar 15 21:35:17 2011
@@ -61,12 +61,12 @@ public class TestDateFilter extends Luce
// filter that should preserve matches
// DateFilter df1 = DateFilter.Before("datefield", now);
- TermRangeFilter df1 = new TermRangeFilter("datefield", DateTools
+ TermRangeFilter df1 = TermRangeFilter.newStringRange("datefield", DateTools
.timeToString(now - 2000, DateTools.Resolution.MILLISECOND), DateTools
.timeToString(now, DateTools.Resolution.MILLISECOND), false, true);
// filter that should discard matches
// DateFilter df2 = DateFilter.Before("datefield", now - 999999);
- TermRangeFilter df2 = new TermRangeFilter("datefield", DateTools
+ TermRangeFilter df2 = TermRangeFilter.newStringRange("datefield", DateTools
.timeToString(0, DateTools.Resolution.MILLISECOND), DateTools
.timeToString(now - 2000, DateTools.Resolution.MILLISECOND), true,
false);
@@ -128,13 +128,13 @@ public class TestDateFilter extends Luce
// filter that should preserve matches
// DateFilter df1 = DateFilter.After("datefield", now);
- TermRangeFilter df1 = new TermRangeFilter("datefield", DateTools
+ TermRangeFilter df1 = TermRangeFilter.newStringRange("datefield", DateTools
.timeToString(now, DateTools.Resolution.MILLISECOND), DateTools
.timeToString(now + 999999, DateTools.Resolution.MILLISECOND), true,
false);
// filter that should discard matches
// DateFilter df2 = DateFilter.After("datefield", now + 999999);
- TermRangeFilter df2 = new TermRangeFilter("datefield", DateTools
+ TermRangeFilter df2 = TermRangeFilter.newStringRange("datefield", DateTools
.timeToString(now + 999999, DateTools.Resolution.MILLISECOND),
DateTools.timeToString(now + 999999999,
DateTools.Resolution.MILLISECOND), false, true);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestExplanations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestExplanations.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestExplanations.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestExplanations.java Tue Mar 15 21:35:17 2011
@@ -52,7 +52,10 @@ public class TestExplanations extends Lu
protected Directory directory;
public static final String KEY = "KEY";
+ // boost on this field is the same as the iterator for the doc
public static final String FIELD = "field";
+ // same contents, but no field boost
+ public static final String ALTFIELD = "alt";
public static final QueryParser qp =
new QueryParser(TEST_VERSION_CURRENT, FIELD, new MockAnalyzer());
@@ -72,7 +75,10 @@ public class TestExplanations extends Lu
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(newField(KEY, ""+i, Field.Store.NO, Field.Index.NOT_ANALYZED));
- doc.add(newField(FIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
+ Field f = newField(FIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED);
+ f.setBoost(i);
+ doc.add(f);
+ doc.add(newField(ALTFIELD, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
writer.addDocument(doc);
}
reader = writer.getReader();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java Tue Mar 15 21:35:17 2011
@@ -187,7 +187,7 @@ public class TestFilteredQuery extends L
* This tests FilteredQuery's rewrite correctness
*/
public void testRangeQuery() throws Exception {
- TermRangeQuery rq = new TermRangeQuery(
+ TermRangeQuery rq = TermRangeQuery.newStringRange(
"sorter", "b", "d", true, true);
Query filteredquery = new FilteredQuery(rq, filter);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java Tue Mar 15 21:35:17 2011
@@ -92,26 +92,18 @@ public class TestMultiTermConstantScore
/** macro for readability */
public static Query csrq(String f, String l, String h, boolean il, boolean ih) {
- TermRangeQuery query = new TermRangeQuery(f, l, h, il, ih);
+ TermRangeQuery query = TermRangeQuery.newStringRange(f, l, h, il, ih);
query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
return query;
}
public static Query csrq(String f, String l, String h, boolean il, boolean ih, MultiTermQuery.RewriteMethod method) {
- TermRangeQuery query = new TermRangeQuery(f, l, h, il, ih);
+ TermRangeQuery query = TermRangeQuery.newStringRange(f, l, h, il, ih);
query.setRewriteMethod(method);
return query;
}
/** macro for readability */
- public static Query csrq(String f, String l, String h, boolean il,
- boolean ih, Collator c) {
- TermRangeQuery query = new TermRangeQuery(f, l, h, il, ih, c);
- query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
- return query;
- }
-
- /** macro for readability */
public static Query cspq(Term prefix) {
PrefixQuery query = new PrefixQuery(prefix);
query.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
@@ -142,15 +134,6 @@ public class TestMultiTermConstantScore
}
@Test
- public void testBasicsRngCollating() throws IOException {
- Collator c = Collator.getInstance(Locale.ENGLISH);
- QueryUtils.check(csrq("data", "1", "6", T, T, c));
- QueryUtils.check(csrq("data", "A", "Z", T, T, c));
- QueryUtils.checkUnequal(csrq("data", "1", "6", T, T, c), csrq("data", "A",
- "Z", T, T, c));
- }
-
- @Test
public void testEqualScores() throws IOException {
// NOTE: uses index build in *this* setUp
@@ -262,7 +245,7 @@ public class TestMultiTermConstantScore
// first do a regular TermRangeQuery which uses term expansion so
// docs with more terms in range get higher scores
- Query rq = new TermRangeQuery("data", "1", "4", T, T);
+ Query rq = TermRangeQuery.newStringRange("data", "1", "4", T, T);
ScoreDoc[] expected = search.search(rq, null, 1000).scoreDocs;
int numHits = expected.length;
@@ -416,92 +399,6 @@ public class TestMultiTermConstantScore
}
@Test
- public void testRangeQueryIdCollating() throws IOException {
- // NOTE: uses index build in *super* setUp
-
- IndexReader reader = signedIndexReader;
- IndexSearcher search = newSearcher(reader);
-
- int medId = ((maxId - minId) / 2);
-
- String minIP = pad(minId);
- String maxIP = pad(maxId);
- String medIP = pad(medId);
-
- int numDocs = reader.numDocs();
-
- assertEquals("num of docs", numDocs, 1 + maxId - minId);
-
- ScoreDoc[] result;
-
- Collator c = Collator.getInstance(Locale.ENGLISH);
-
- // test id, bounded on both ends
-
- result = search.search(csrq("id", minIP, maxIP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("find all", numDocs, result.length);
-
- result = search.search(csrq("id", minIP, maxIP, T, F, c), null, numDocs).scoreDocs;
- assertEquals("all but last", numDocs - 1, result.length);
-
- result = search.search(csrq("id", minIP, maxIP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("all but first", numDocs - 1, result.length);
-
- result = search.search(csrq("id", minIP, maxIP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("all but ends", numDocs - 2, result.length);
-
- result = search.search(csrq("id", medIP, maxIP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("med and up", 1 + maxId - medId, result.length);
-
- result = search.search(csrq("id", minIP, medIP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("up to med", 1 + medId - minId, result.length);
-
- // unbounded id
-
- result = search.search(csrq("id", minIP, null, T, F, c), null, numDocs).scoreDocs;
- assertEquals("min and up", numDocs, result.length);
-
- result = search.search(csrq("id", null, maxIP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("max and down", numDocs, result.length);
-
- result = search.search(csrq("id", minIP, null, F, F, c), null, numDocs).scoreDocs;
- assertEquals("not min, but up", numDocs - 1, result.length);
-
- result = search.search(csrq("id", null, maxIP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("not max, but down", numDocs - 1, result.length);
-
- result = search.search(csrq("id", medIP, maxIP, T, F, c), null, numDocs).scoreDocs;
- assertEquals("med and up, not max", maxId - medId, result.length);
-
- result = search.search(csrq("id", minIP, medIP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("not min, up to med", medId - minId, result.length);
-
- // very small sets
-
- result = search.search(csrq("id", minIP, minIP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("min,min,F,F,c", 0, result.length);
- result = search.search(csrq("id", medIP, medIP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("med,med,F,F,c", 0, result.length);
- result = search.search(csrq("id", maxIP, maxIP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("max,max,F,F,c", 0, result.length);
-
- result = search.search(csrq("id", minIP, minIP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("min,min,T,T,c", 1, result.length);
- result = search.search(csrq("id", null, minIP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("nul,min,F,T,c", 1, result.length);
-
- result = search.search(csrq("id", maxIP, maxIP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("max,max,T,T,c", 1, result.length);
- result = search.search(csrq("id", maxIP, null, T, F, c), null, numDocs).scoreDocs;
- assertEquals("max,nul,T,T,c", 1, result.length);
-
- result = search.search(csrq("id", medIP, medIP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("med,med,T,T,c", 1, result.length);
-
- search.close();
- }
-
- @Test
public void testRangeQueryRand() throws IOException {
// NOTE: uses index build in *super* setUp
@@ -564,151 +461,4 @@ public class TestMultiTermConstantScore
search.close();
}
-
- @Test
- public void testRangeQueryRandCollating() throws IOException {
- // NOTE: uses index build in *super* setUp
-
- // using the unsigned index because collation seems to ignore hyphens
- IndexReader reader = unsignedIndexReader;
- IndexSearcher search = newSearcher(reader);
-
- String minRP = pad(unsignedIndexDir.minR);
- String maxRP = pad(unsignedIndexDir.maxR);
-
- int numDocs = reader.numDocs();
-
- assertEquals("num of docs", numDocs, 1 + maxId - minId);
-
- ScoreDoc[] result;
-
- Collator c = Collator.getInstance(Locale.ENGLISH);
-
- // test extremes, bounded on both ends
-
- result = search.search(csrq("rand", minRP, maxRP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("find all", numDocs, result.length);
-
- result = search.search(csrq("rand", minRP, maxRP, T, F, c), null, numDocs).scoreDocs;
- assertEquals("all but biggest", numDocs - 1, result.length);
-
- result = search.search(csrq("rand", minRP, maxRP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("all but smallest", numDocs - 1, result.length);
-
- result = search.search(csrq("rand", minRP, maxRP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("all but extremes", numDocs - 2, result.length);
-
- // unbounded
-
- result = search.search(csrq("rand", minRP, null, T, F, c), null, numDocs).scoreDocs;
- assertEquals("smallest and up", numDocs, result.length);
-
- result = search.search(csrq("rand", null, maxRP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("biggest and down", numDocs, result.length);
-
- result = search.search(csrq("rand", minRP, null, F, F, c), null, numDocs).scoreDocs;
- assertEquals("not smallest, but up", numDocs - 1, result.length);
-
- result = search.search(csrq("rand", null, maxRP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("not biggest, but down", numDocs - 1, result.length);
-
- // very small sets
-
- result = search.search(csrq("rand", minRP, minRP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("min,min,F,F,c", 0, result.length);
- result = search.search(csrq("rand", maxRP, maxRP, F, F, c), null, numDocs).scoreDocs;
- assertEquals("max,max,F,F,c", 0, result.length);
-
- result = search.search(csrq("rand", minRP, minRP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("min,min,T,T,c", 1, result.length);
- result = search.search(csrq("rand", null, minRP, F, T, c), null, numDocs).scoreDocs;
- assertEquals("nul,min,F,T,c", 1, result.length);
-
- result = search.search(csrq("rand", maxRP, maxRP, T, T, c), null, numDocs).scoreDocs;
- assertEquals("max,max,T,T,c", 1, result.length);
- result = search.search(csrq("rand", maxRP, null, T, F, c), null, numDocs).scoreDocs;
- assertEquals("max,nul,T,T,c", 1, result.length);
-
- search.close();
- }
-
- @Test
- public void testFarsi() throws Exception {
-
- /* build an index */
- Directory farsiIndex = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random, farsiIndex, new MockAnalyzer(MockTokenizer.SIMPLE, true));
- Document doc = new Document();
- doc.add(newField("content", "\u0633\u0627\u0628", Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- doc
- .add(newField("body", "body", Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- writer.addDocument(doc);
-
- IndexReader reader = writer.getReader();
- writer.close();
-
- IndexSearcher search = newSearcher(reader);
-
- // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
- // RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
- // characters properly.
- Collator c = Collator.getInstance(new Locale("ar"));
-
- // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
- // orders the U+0698 character before the U+0633 character, so the single
- // index Term below should NOT be returned by a ConstantScoreRangeQuery
- // with a Farsi Collator (or an Arabic one for the case when Farsi is
- // not supported).
- ScoreDoc[] result = search.search(csrq("content", "\u062F", "\u0698", T, T,
- c), null, 1000).scoreDocs;
- assertEquals("The index Term should not be included.", 0, result.length);
-
- result = search.search(csrq("content", "\u0633", "\u0638", T, T, c), null,
- 1000).scoreDocs;
- assertEquals("The index Term should be included.", 1, result.length);
- search.close();
- reader.close();
- farsiIndex.close();
- }
-
- @Test
- public void testDanish() throws Exception {
-
- /* build an index */
- Directory danishIndex = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random, danishIndex, new MockAnalyzer(MockTokenizer.SIMPLE, true));
-
- // Danish collation orders the words below in the given order
- // (example taken from TestSort.testInternationalSort() ).
- String[] words = { "H\u00D8T", "H\u00C5T", "MAND" };
- for (int docnum = 0 ; docnum < words.length ; ++docnum) {
- Document doc = new Document();
- doc.add(newField("content", words[docnum],
- Field.Store.YES, Field.Index.NOT_ANALYZED));
- doc.add(newField("body", "body",
- Field.Store.YES, Field.Index.NOT_ANALYZED));
- writer.addDocument(doc);
- }
- IndexReader reader = writer.getReader();
- writer.close();
-
- IndexSearcher search = newSearcher(reader);
-
- Collator c = Collator.getInstance(new Locale("da", "dk"));
-
- // Unicode order would not include "H\u00C5T" in [ "H\u00D8T", "MAND" ],
- // but Danish collation does.
- ScoreDoc[] result = search.search
- (csrq("content", "H\u00D8T", "MAND", F, F, c), null, 1000).scoreDocs;
- assertEquals("The index Term should be included.", 1, result.length);
-
- result = search.search
- (csrq("content", "H\u00C5T", "MAND", F, F, c), null, 1000).scoreDocs;
- assertEquals("The index Term should not be included.", 0, result.length);
- search.close();
- reader.close();
- danishIndex.close();
- }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiTermQueryRewrites.java Tue Mar 15 21:35:17 2011
@@ -112,7 +112,7 @@ public class TestMultiTermQueryRewrites
}
private void checkDuplicateTerms(MultiTermQuery.RewriteMethod method) throws Exception {
- final MultiTermQuery mtq = new TermRangeQuery("data", "2", "7", true, true);
+ final MultiTermQuery mtq = TermRangeQuery.newStringRange("data", "2", "7", true, true);
mtq.setRewriteMethod(method);
final Query q1 = searcher.rewrite(mtq);
final Query q2 = multiSearcher.rewrite(mtq);
@@ -158,7 +158,7 @@ public class TestMultiTermQueryRewrites
final MultiTermQuery mtq = new MultiTermQuery("data") {
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
- return new TermRangeTermsEnum(terms.iterator(), "2", "7", true, true, null) {
+ return new TermRangeTermsEnum(terms.iterator(), new BytesRef("2"), new BytesRef("7"), true, true) {
final BoostAttribute boostAtt =
attributes().addAttribute(BoostAttribute.class);
@@ -203,7 +203,7 @@ public class TestMultiTermQueryRewrites
// default gets restored automatically by LuceneTestCase:
BooleanQuery.setMaxClauseCount(3);
- final MultiTermQuery mtq = new TermRangeQuery("data", "2", "7", true, true);
+ final MultiTermQuery mtq = TermRangeQuery.newStringRange("data", "2", "7", true, true);
mtq.setRewriteMethod(method);
try {
multiSearcherDupls.rewrite(mtq);
@@ -219,7 +219,7 @@ public class TestMultiTermQueryRewrites
// default gets restored automatically by LuceneTestCase:
BooleanQuery.setMaxClauseCount(3);
- final MultiTermQuery mtq = new TermRangeQuery("data", "2", "7", true, true);
+ final MultiTermQuery mtq = TermRangeQuery.newStringRange("data", "2", "7", true, true);
mtq.setRewriteMethod(method);
multiSearcherDupls.rewrite(mtq);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiValuedNumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiValuedNumericRangeQuery.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiValuedNumericRangeQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiValuedNumericRangeQuery.java Tue Mar 15 21:35:17 2011
@@ -67,7 +67,7 @@ public class TestMultiValuedNumericRange
if (lower>upper) {
int a=lower; lower=upper; upper=a;
}
- TermRangeQuery cq=new TermRangeQuery("asc", format.format(lower), format.format(upper), true, true);
+ TermRangeQuery cq=TermRangeQuery.newStringRange("asc", format.format(lower), format.format(upper), true, true);
NumericRangeQuery<Integer> tq=NumericRangeQuery.newIntRange("trie", lower, upper, true, true);
TopDocs trTopDocs = searcher.search(cq, 1);
TopDocs nrTopDocs = searcher.search(tq, 1);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=1081952&r1=1081951&r2=1081952&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Tue Mar 15 21:35:17 2011
@@ -344,12 +344,10 @@ public class TestNumericRangeQuery32 ext
final BytesRef lowerBytes = new BytesRef(NumericUtils.BUF_SIZE_INT), upperBytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
NumericUtils.intToPrefixCoded(lower, 0, lowerBytes);
NumericUtils.intToPrefixCoded(upper, 0, upperBytes);
- // TODO: when new TermRange ctors with BytesRef available, use them and do not convert to string!
- final String lowerString = lowerBytes.utf8ToString(), upperString = upperBytes.utf8ToString();
// test inclusive range
NumericRangeQuery<Integer> tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, true);
- TermRangeQuery cq=new TermRangeQuery(field, lowerString, upperString, true, true);
+ TermRangeQuery cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, true);
TopDocs tTopDocs = searcher.search(tq, 1);
TopDocs cTopDocs = searcher.search(cq, 1);
assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -357,7 +355,7 @@ public class TestNumericRangeQuery32 ext
termCountC += cq.getTotalNumberOfTerms();
// test exclusive range
tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, false);
- cq=new TermRangeQuery(field, lowerString, upperString, false, false);
+ cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, false);
tTopDocs = searcher.search(tq, 1);
cTopDocs = searcher.search(cq, 1);
assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -365,7 +363,7 @@ public class TestNumericRangeQuery32 ext
termCountC += cq.getTotalNumberOfTerms();
// test left exclusive range
tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, false, true);
- cq=new TermRangeQuery(field, lowerString, upperString, false, true);
+ cq=new TermRangeQuery(field, lowerBytes, upperBytes, false, true);
tTopDocs = searcher.search(tq, 1);
cTopDocs = searcher.search(cq, 1);
assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
@@ -373,7 +371,7 @@ public class TestNumericRangeQuery32 ext
termCountC += cq.getTotalNumberOfTerms();
// test right exclusive range
tq=NumericRangeQuery.newIntRange(field, precisionStep, lower, upper, true, false);
- cq=new TermRangeQuery(field, lowerString, upperString, true, false);
+ cq=new TermRangeQuery(field, lowerBytes, upperBytes, true, false);
tTopDocs = searcher.search(tq, 1);
cTopDocs = searcher.search(cq, 1);
assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );