You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/09 15:19:39 UTC
svn commit: r1101016 [5/10] - in /lucene/dev/branches/bulkpostings: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/solr/ dev-tools/maven/lucene/contrib/ant/
dev-tools/maven/lucene/contrib/db/bdb-je/ dev-tools/maven/luce...
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Mon May 9 13:19:28 2011
@@ -529,10 +529,9 @@ public class TestBackwardsCompatibility
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
int contentFieldIndex = -1;
- for(int i=0;i<fieldInfos.size();i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
+ for (FieldInfo fi : fieldInfos) {
if (fi.name.equals("content")) {
- contentFieldIndex = i;
+ contentFieldIndex = fi.number;
break;
}
}
@@ -544,7 +543,8 @@ public class TestBackwardsCompatibility
"_0_1.del",
"_0_1.s" + contentFieldIndex,
"segments_2",
- "segments.gen"};
+ "segments.gen",
+ "1.fnx"};
String[] actual = dir.listAll();
Arrays.sort(expected);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java Mon May 9 13:19:28 2011
@@ -23,14 +23,14 @@ import java.util.HashSet;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PostingsConsumer;
-import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.index.codecs.TermStats;
+import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.search.DocIdSetIterator;
@@ -80,7 +80,7 @@ public class TestCodecs extends LuceneTe
public FieldData(final String name, final FieldInfos fieldInfos, final TermData[] terms, final boolean omitTF, final boolean storePayloads) {
this.omitTF = omitTF;
this.storePayloads = storePayloads;
- fieldInfos.add(name, true);
+ fieldInfos.addOrUpdate(name, true);
fieldInfo = fieldInfos.fieldInfo(name);
fieldInfo.omitTermFreqAndPositions = omitTF;
fieldInfo.storePayloads = storePayloads;
@@ -238,8 +238,9 @@ public class TestCodecs extends LuceneTe
final FieldData[] fields = new FieldData[] {field};
final Directory dir = newDirectory();
+ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone();
this.write(fieldInfos, dir, fields, true);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos.hasVectors(), clonedFieldInfos);
si.setHasProx(false);
final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -289,8 +290,10 @@ public class TestCodecs extends LuceneTe
if (VERBOSE) {
System.out.println("TEST: now write postings");
}
+
+ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone();
this.write(fieldInfos, dir, fields, false);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos.hasVectors(), clonedFieldInfos);
if (VERBOSE) {
System.out.println("TEST: now read postings");
@@ -439,8 +442,8 @@ public class TestCodecs extends LuceneTe
for(int iter=0;iter<NUM_TEST_ITER;iter++) {
final FieldData field = fields[TestCodecs.random.nextInt(fields.length)];
final TermsEnum termsEnum = termsDict.terms(field.fieldInfo.name).iterator();
-
- if (si.getSegmentCodecs().codecs[field.fieldInfo.codecId] instanceof PreFlexCodec) {
+ assertTrue(field.fieldInfo.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID);
+ if (si.getSegmentCodecs().codecs[field.fieldInfo.getCodecId()] instanceof PreFlexCodec) {
// code below expects unicode sort order
continue;
}
@@ -588,13 +591,14 @@ public class TestCodecs extends LuceneTe
private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields, boolean allowPreFlex) throws Throwable {
final int termIndexInterval = _TestUtil.nextInt(random, 13, 27);
- final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault());
+ final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false);
final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, codecInfo, null);
final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
Arrays.sort(fields);
for (final FieldData field : fields) {
- if (!allowPreFlex && codecInfo.codecs[field.fieldInfo.codecId] instanceof PreFlexCodec) {
+ assertTrue(field.fieldInfo.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID);
+ if (!allowPreFlex && codecInfo.codecs[field.fieldInfo.getCodecId()] instanceof PreFlexCodec) {
// code below expects unicode sort order
continue;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java Mon May 9 13:19:28 2011
@@ -202,10 +202,10 @@ public class TestDoc extends LuceneTestC
merger.merge();
r1.close();
r2.close();
-
+ final FieldInfos fieldInfos = merger.fieldInfos();
final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
- false, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(),
- merger.fieldInfos().hasVectors());
+ false, fieldInfos.hasProx(), merger.getSegmentCodecs(),
+ fieldInfos.hasVectors(), fieldInfos);
if (useCompoundFile) {
Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon May 9 13:19:28 2011
@@ -25,20 +25,20 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.BytesRef;
public class TestDocumentWriter extends LuceneTestCase {
private Directory dir;
@@ -98,8 +98,7 @@ public class TestDocumentWriter extends
// test that the norms are not present in the segment if
// omitNorms is true
- for (int i = 0; i < reader.core.fieldInfos.size(); i++) {
- FieldInfo fi = reader.core.fieldInfos.fieldInfo(i);
+ for (FieldInfo fi : reader.core.fieldInfos) {
if (fi.isIndexed) {
assertTrue(fi.omitNorms == !reader.hasNorms(fi.name));
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Mon May 9 13:19:28 2011
@@ -18,11 +18,13 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
import java.io.IOException;
+import java.util.Arrays;
//import org.cnlp.utils.properties.ResourceBundleHelper;
@@ -36,44 +38,124 @@ public class TestFieldInfos extends Luce
DocHelper.setupDoc(testDoc);
}
- public void test() throws IOException {
- //Positive test of FieldInfos
+ public FieldInfos createAndWriteFieldInfos(Directory dir, String filename) throws IOException{
+ //Positive test of FieldInfos
assertTrue(testDoc != null);
FieldInfos fieldInfos = new FieldInfos();
- fieldInfos.add(testDoc);
+ _TestUtil.add(testDoc, fieldInfos);
//Since the complement is stored as well in the fields map
assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
- Directory dir = newDirectory();
- String name = "testFile";
- IndexOutput output = dir.createOutput(name);
+
+
+ IndexOutput output = dir.createOutput(filename);
assertTrue(output != null);
//Use a RAMOutputStream
-
- fieldInfos.write(output);
- output.close();
- assertTrue(dir.fileLength(name) > 0);
- FieldInfos readIn = new FieldInfos(dir, name);
- assertTrue(fieldInfos.size() == readIn.size());
- FieldInfo info = readIn.fieldInfo("textField1");
- assertTrue(info != null);
- assertTrue(info.storeTermVector == false);
- assertTrue(info.omitNorms == false);
-
- info = readIn.fieldInfo("textField2");
- assertTrue(info != null);
- assertTrue(info.storeTermVector == true);
- assertTrue(info.omitNorms == false);
-
- info = readIn.fieldInfo("textField3");
- assertTrue(info != null);
- assertTrue(info.storeTermVector == false);
- assertTrue(info.omitNorms == true);
-
- info = readIn.fieldInfo("omitNorms");
- assertTrue(info != null);
- assertTrue(info.storeTermVector == false);
- assertTrue(info.omitNorms == true);
+
+ fieldInfos.write(output);
+ output.close();
+ return fieldInfos;
+ }
+ public void test() throws IOException {
+ String name = "testFile";
+ Directory dir = newDirectory();
+ FieldInfos fieldInfos = createAndWriteFieldInfos(dir, name);
+ assertTrue(dir.fileLength(name) > 0);
+ FieldInfos readIn = new FieldInfos(dir, name);
+ assertTrue(fieldInfos.size() == readIn.size());
+ FieldInfo info = readIn.fieldInfo("textField1");
+ assertTrue(info != null);
+ assertTrue(info.storeTermVector == false);
+ assertTrue(info.omitNorms == false);
+
+ info = readIn.fieldInfo("textField2");
+ assertTrue(info != null);
+ assertTrue(info.storeTermVector == true);
+ assertTrue(info.omitNorms == false);
+
+ info = readIn.fieldInfo("textField3");
+ assertTrue(info != null);
+ assertTrue(info.storeTermVector == false);
+ assertTrue(info.omitNorms == true);
+
+ info = readIn.fieldInfo("omitNorms");
+ assertTrue(info != null);
+ assertTrue(info.storeTermVector == false);
+ assertTrue(info.omitNorms == true);
- dir.close();
+ dir.close();
+ }
+
+ public void testReadOnly() throws IOException {
+ String name = "testFile";
+ Directory dir = newDirectory();
+ FieldInfos fieldInfos = createAndWriteFieldInfos(dir, name);
+ FieldInfos readOnly = new FieldInfos(dir, name);
+ assertReadOnly(readOnly, fieldInfos);
+ FieldInfos readOnlyClone = (FieldInfos)readOnly.clone();
+ assertNotSame(readOnly, readOnlyClone);
+ // clone is also read only - no global field map
+ assertReadOnly(readOnlyClone, fieldInfos);
+ dir.close();
+ }
+
+ private void assertReadOnly(FieldInfos readOnly, FieldInfos modifiable) {
+ assertTrue(readOnly.isReadOnly());
+ assertFalse(modifiable.isReadOnly());
+ try {
+ readOnly.add(modifiable.fieldInfo(0));
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ try {
+ readOnly.addOrUpdate("bogus", random.nextBoolean());
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+ try {
+ readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean());
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+ try {
+ readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
+ random.nextBoolean(), random.nextBoolean());
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+ try {
+ readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
+ random.nextBoolean(), random.nextBoolean(), random.nextBoolean());
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+ try {
+ readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
+ random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
+ random.nextBoolean(), random.nextBoolean());
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+ try {
+ readOnly.addOrUpdate(Arrays.asList("a", "b", "c"), random.nextBoolean());
+ fail("instance should be read only");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ assertEquals(modifiable.size(), readOnly.size());
+ // assert we can iterate
+ for (FieldInfo fi : readOnly) {
+ assertEquals(fi.name, modifiable.fieldName(fi.number));
+ }
+
}
+
+
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Mon May 9 13:19:28 2011
@@ -49,7 +49,7 @@ public class TestFieldsReader extends Lu
super.setUp();
fieldInfos = new FieldInfos();
DocHelper.setupDoc(testDoc);
- fieldInfos.add(testDoc);
+ _TestUtil.add(testDoc, fieldInfos);
dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy());
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Mon May 9 13:19:28 2011
@@ -92,10 +92,9 @@ public class TestIndexFileDeleter extend
CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
int contentFieldIndex = -1;
- for(i=0;i<fieldInfos.size();i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
+ for (FieldInfo fi : fieldInfos) {
if (fi.name.equals("content")) {
- contentFieldIndex = i;
+ contentFieldIndex = fi.number;
break;
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexInput.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexInput.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexInput.java Mon May 9 13:19:28 2011
@@ -19,51 +19,60 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMDirectory;
import java.io.IOException;
public class TestIndexInput extends LuceneTestCase {
- public void testRead() throws IOException {
- IndexInput is = new MockIndexInput(new byte[] {
- (byte) 0x80, 0x01,
- (byte) 0xFF, 0x7F,
- (byte) 0x80, (byte) 0x80, 0x01,
- (byte) 0x81, (byte) 0x80, 0x01,
- 0x06, 'L', 'u', 'c', 'e', 'n', 'e',
-
- // 2-byte UTF-8 (U+00BF "INVERTED QUESTION MARK")
- 0x02, (byte) 0xC2, (byte) 0xBF,
- 0x0A, 'L', 'u', (byte) 0xC2, (byte) 0xBF,
- 'c', 'e', (byte) 0xC2, (byte) 0xBF,
- 'n', 'e',
-
- // 3-byte UTF-8 (U+2620 "SKULL AND CROSSBONES")
- 0x03, (byte) 0xE2, (byte) 0x98, (byte) 0xA0,
- 0x0C, 'L', 'u', (byte) 0xE2, (byte) 0x98, (byte) 0xA0,
- 'c', 'e', (byte) 0xE2, (byte) 0x98, (byte) 0xA0,
- 'n', 'e',
-
- // surrogate pairs
- // (U+1D11E "MUSICAL SYMBOL G CLEF")
- // (U+1D160 "MUSICAL SYMBOL EIGHTH NOTE")
- 0x04, (byte) 0xF0, (byte) 0x9D, (byte) 0x84, (byte) 0x9E,
- 0x08, (byte) 0xF0, (byte) 0x9D, (byte) 0x84, (byte) 0x9E,
- (byte) 0xF0, (byte) 0x9D, (byte) 0x85, (byte) 0xA0,
- 0x0E, 'L', 'u',
- (byte) 0xF0, (byte) 0x9D, (byte) 0x84, (byte) 0x9E,
- 'c', 'e',
- (byte) 0xF0, (byte) 0x9D, (byte) 0x85, (byte) 0xA0,
- 'n', 'e',
-
- // null bytes
- 0x01, 0x00,
- 0x08, 'L', 'u', 0x00, 'c', 'e', 0x00, 'n', 'e',
- });
-
+
+ static final byte[] READ_TEST_BYTES = new byte[] {
+ (byte) 0x80, 0x01,
+ (byte) 0xFF, 0x7F,
+ (byte) 0x80, (byte) 0x80, 0x01,
+ (byte) 0x81, (byte) 0x80, 0x01,
+ (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0x07,
+ (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0x07,
+ (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0x7F,
+ 0x06, 'L', 'u', 'c', 'e', 'n', 'e',
+
+ // 2-byte UTF-8 (U+00BF "INVERTED QUESTION MARK")
+ 0x02, (byte) 0xC2, (byte) 0xBF,
+ 0x0A, 'L', 'u', (byte) 0xC2, (byte) 0xBF,
+ 'c', 'e', (byte) 0xC2, (byte) 0xBF,
+ 'n', 'e',
+
+ // 3-byte UTF-8 (U+2620 "SKULL AND CROSSBONES")
+ 0x03, (byte) 0xE2, (byte) 0x98, (byte) 0xA0,
+ 0x0C, 'L', 'u', (byte) 0xE2, (byte) 0x98, (byte) 0xA0,
+ 'c', 'e', (byte) 0xE2, (byte) 0x98, (byte) 0xA0,
+ 'n', 'e',
+
+ // surrogate pairs
+ // (U+1D11E "MUSICAL SYMBOL G CLEF")
+ // (U+1D160 "MUSICAL SYMBOL EIGHTH NOTE")
+ 0x04, (byte) 0xF0, (byte) 0x9D, (byte) 0x84, (byte) 0x9E,
+ 0x08, (byte) 0xF0, (byte) 0x9D, (byte) 0x84, (byte) 0x9E,
+ (byte) 0xF0, (byte) 0x9D, (byte) 0x85, (byte) 0xA0,
+ 0x0E, 'L', 'u',
+ (byte) 0xF0, (byte) 0x9D, (byte) 0x84, (byte) 0x9E,
+ 'c', 'e',
+ (byte) 0xF0, (byte) 0x9D, (byte) 0x85, (byte) 0xA0,
+ 'n', 'e',
+
+ // null bytes
+ 0x01, 0x00,
+ 0x08, 'L', 'u', 0x00, 'c', 'e', 0x00, 'n', 'e',
+ };
+
+ private void checkReads(IndexInput is) throws IOException {
assertEquals(128,is.readVInt());
assertEquals(16383,is.readVInt());
assertEquals(16384,is.readVInt());
assertEquals(16385,is.readVInt());
+ assertEquals(Integer.MAX_VALUE, is.readVInt());
+ assertEquals((long) Integer.MAX_VALUE, is.readVLong());
+ assertEquals(Long.MAX_VALUE, is.readVLong());
assertEquals("Lucene",is.readString());
assertEquals("\u00BF",is.readString());
@@ -79,4 +88,24 @@ public class TestIndexInput extends Luce
assertEquals("\u0000",is.readString());
assertEquals("Lu\u0000ce\u0000ne",is.readString());
}
+
+ // this test only checks BufferedIndexInput because MockIndexInput extends BufferedIndexInput
+ public void testBufferedIndexInputRead() throws IOException {
+ final IndexInput is = new MockIndexInput(READ_TEST_BYTES);
+ checkReads(is);
+ is.close();
+ }
+
+ // this test checks the raw IndexInput methods as it uses RAMIndexInput which extends IndexInput directly
+ public void testRawIndexInputRead() throws IOException {
+ final RAMDirectory dir = new RAMDirectory();
+ final IndexOutput os = dir.createOutput("foo");
+ os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length);
+ os.close();
+ final IndexInput is = dir.openInput("foo");
+ checkReads(is);
+ is.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Mon May 9 13:19:28 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.document.Field.
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.SegmentReader.Norm;
import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -41,17 +42,22 @@ import org.apache.lucene.util.LuceneTest
*/
public class TestIndexReaderCloneNorms extends LuceneTestCase {
- private class SimilarityOne extends DefaultSimilarity {
+ private class SimilarityProviderOne extends DefaultSimilarityProvider {
@Override
- public float computeNorm(FieldInvertState state) {
- // diable length norm
- return state.getBoost();
- }
+ public Similarity get(String field) {
+ return new DefaultSimilarity() {
+ @Override
+ public float computeNorm(FieldInvertState state) {
+ // diable length norm
+ return state.getBoost();
+ }
+ };
+ }
}
private static final int NUM_FIELDS = 10;
- private SimilarityProvider similarityOne;
+ private SimilarityProvider similarityProviderOne;
private Analyzer anlzr;
@@ -68,7 +74,7 @@ public class TestIndexReaderCloneNorms e
@Override
public void setUp() throws Exception {
super.setUp();
- similarityOne = new SimilarityOne();
+ similarityProviderOne = new SimilarityProviderOne();
anlzr = new MockAnalyzer();
}
@@ -249,7 +255,7 @@ public class TestIndexReaderCloneNorms e
}
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(5).setSimilarityProvider(similarityOne).setMergePolicy(newLogMergePolicy()));
+ .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
@@ -307,7 +313,7 @@ public class TestIndexReaderCloneNorms e
throws IOException {
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5).setSimilarityProvider(similarityOne).setMergePolicy(newLogMergePolicy());
+ .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy());
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Mon May 9 13:19:28 2011
@@ -686,6 +686,122 @@ public class TestIndexWriter extends Luc
dir.close();
}
+ // Make sure it's OK to change RAM buffer size and
+ // maxBufferedDocs in a write session
+ public void testChangingRAMBuffer() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+
+ int lastFlushCount = -1;
+ for(int j=1;j<52;j++) {
+ Document doc = new Document();
+ doc.add(new Field("field", "aaa" + j, Field.Store.YES, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ _TestUtil.syncConcurrentMerges(writer);
+ int flushCount = writer.getFlushCount();
+ if (j == 1)
+ lastFlushCount = flushCount;
+ else if (j < 10)
+ // No new files should be created
+ assertEquals(flushCount, lastFlushCount);
+ else if (10 == j) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (j < 20) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (20 == j) {
+ writer.getConfig().setRAMBufferSizeMB(16);
+ writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 30) {
+ assertEquals(flushCount, lastFlushCount);
+ } else if (30 == j) {
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (j < 40) {
+ assertTrue(flushCount> lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (40 == j) {
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 50) {
+ assertEquals(flushCount, lastFlushCount);
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (50 == j) {
+ assertTrue(flushCount > lastFlushCount);
+ }
+ }
+ writer.close();
+ dir.close();
+ }
+
+ public void testChangingRAMBuffer2() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setMaxBufferedDeleteTerms(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+
+ for(int j=1;j<52;j++) {
+ Document doc = new Document();
+ doc.add(new Field("field", "aaa" + j, Field.Store.YES, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+
+ int lastFlushCount = -1;
+ for(int j=1;j<52;j++) {
+ writer.deleteDocuments(new Term("field", "aaa" + j));
+ _TestUtil.syncConcurrentMerges(writer);
+ int flushCount = writer.getFlushCount();
+ if (j == 1)
+ lastFlushCount = flushCount;
+ else if (j < 10) {
+ // No new files should be created
+ assertEquals(flushCount, lastFlushCount);
+ } else if (10 == j) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDeleteTerms(1);
+ } else if (j < 20) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (20 == j) {
+ writer.getConfig().setRAMBufferSizeMB(16);
+ writer.getConfig().setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 30) {
+ assertEquals(flushCount, lastFlushCount);
+ } else if (30 == j) {
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ writer.getConfig().setMaxBufferedDeleteTerms(1);
+ } else if (j < 40) {
+ assertTrue(flushCount> lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (40 == j) {
+ writer.getConfig().setMaxBufferedDeleteTerms(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 50) {
+ assertEquals(flushCount, lastFlushCount);
+ writer.getConfig().setMaxBufferedDeleteTerms(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (50 == j) {
+ assertTrue(flushCount > lastFlushCount);
+ }
+ }
+ writer.close();
+ dir.close();
+ }
+
public void testDiverseDocs() throws IOException {
MockDirectoryWrapper dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setRAMBufferSizeMB(0.5));
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Mon May 9 13:19:28 2011
@@ -26,14 +26,14 @@ import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.index.DocumentsWriter.IndexingChain;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
public class TestIndexWriterConfig extends LuceneTestCase {
- private static final class MySimilarity extends DefaultSimilarity {
+ private static final class MySimilarityProvider extends DefaultSimilarityProvider {
// Does not implement anything - used only for type checking on IndexWriterConfig.
}
@@ -177,8 +177,8 @@ public class TestIndexWriterConfig exten
// Test Similarity:
// we shouldnt assert what the default is, just that its not null.
assertTrue(IndexSearcher.getDefaultSimilarityProvider() == conf.getSimilarityProvider());
- conf.setSimilarityProvider(new MySimilarity());
- assertEquals(MySimilarity.class, conf.getSimilarityProvider().getClass());
+ conf.setSimilarityProvider(new MySimilarityProvider());
+ assertEquals(MySimilarityProvider.class, conf.getSimilarityProvider().getClass());
conf.setSimilarityProvider(null);
assertTrue(IndexSearcher.getDefaultSimilarityProvider() == conf.getSimilarityProvider());
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon May 9 13:19:28 2011
@@ -33,7 +33,9 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.Failure;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -780,7 +782,6 @@ public class TestIndexWriterExceptions e
}
}
}
-
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
assertTrue(failure.didFail);
failure.clearDoFail();
@@ -794,53 +795,79 @@ public class TestIndexWriterExceptions e
private static class FailOnlyInCommit extends MockDirectoryWrapper.Failure {
- boolean fail1, fail2;
+ boolean failOnCommit, failOnDeleteFile;
+ private final boolean dontFailDuringGlobalFieldMap;
+ private static final String PREPARE_STAGE = "prepareCommit";
+ private static final String FINISH_STAGE = "finishCommit";
+ private final String stage;
+
+ public FailOnlyInCommit(boolean dontFailDuringGlobalFieldMap, String stage) {
+ this.dontFailDuringGlobalFieldMap = dontFailDuringGlobalFieldMap;
+ this.stage = stage;
+ }
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
StackTraceElement[] trace = new Exception().getStackTrace();
boolean isCommit = false;
boolean isDelete = false;
+ boolean isInGlobalFieldMap = false;
for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "prepareCommit".equals(trace[i].getMethodName()))
+ if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
isCommit = true;
if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
isDelete = true;
+ if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName()))
+ isInGlobalFieldMap = true;
+
+ }
+ if (isInGlobalFieldMap && dontFailDuringGlobalFieldMap) {
+ isCommit = false;
}
-
if (isCommit) {
if (!isDelete) {
- fail1 = true;
+ failOnCommit = true;
throw new RuntimeException("now fail first");
} else {
- fail2 = true;
+ failOnDeleteFile = true;
throw new IOException("now fail during delete");
}
}
}
}
- // LUCENE-1214
public void testExceptionsDuringCommit() throws Throwable {
- MockDirectoryWrapper dir = newDirectory();
- FailOnlyInCommit failure = new FailOnlyInCommit();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- w.addDocument(doc);
- dir.failOn(failure);
- try {
- w.close();
- fail();
- } catch (IOException ioe) {
- fail("expected only RuntimeException");
- } catch (RuntimeException re) {
- // Expected
+ FailOnlyInCommit[] failures = new FailOnlyInCommit[] {
+ // LUCENE-1214
+ new FailOnlyInCommit(false, FailOnlyInCommit.PREPARE_STAGE), // fail during global field map is written
+ new FailOnlyInCommit(true, FailOnlyInCommit.PREPARE_STAGE), // fail after global field map is written
+ new FailOnlyInCommit(false, FailOnlyInCommit.FINISH_STAGE) // fail while running finishCommit
+ };
+
+ for (FailOnlyInCommit failure : failures) {
+ MockDirectoryWrapper dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ w.addDocument(doc);
+ dir.failOn(failure);
+ try {
+ w.close();
+ fail();
+ } catch (IOException ioe) {
+ fail("expected only RuntimeException");
+ } catch (RuntimeException re) {
+ // Expected
+ }
+ assertTrue(dir.fileExists("1.fnx"));
+ assertTrue(failure.failOnCommit && failure.failOnDeleteFile);
+ w.rollback();
+ assertFalse(dir.fileExists("1.fnx"));
+ // FIXME: on windows, this often fails! assertEquals(0, dir.listAll().length);
+ dir.close();
}
- assertTrue(failure.fail1 && failure.fail2);
- w.rollback();
- dir.close();
}
public void testOptimizeExceptions() throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Mon May 9 13:19:28 2011
@@ -131,7 +131,8 @@ public class TestIndexWriterMergePolicy
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer())
- .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy()));
+ .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
+ .setMergeScheduler(new SerialMergeScheduler()));
// leftmost* segment has 1 doc
// rightmost* segment has 100 docs
@@ -144,8 +145,8 @@ public class TestIndexWriterMergePolicy
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
new MockAnalyzer()).setOpenMode(
- OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(
- new LogDocMergePolicy()));
+ OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
+ .setMergeScheduler(new SerialMergeScheduler()));
}
writer.close();
@@ -153,7 +154,7 @@ public class TestIndexWriterMergePolicy
ldmp.setMergeFactor(10);
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
new MockAnalyzer()).setOpenMode(
- OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
+ OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new SerialMergeScheduler()));
// merge policy only fixes segments on levels where merges
// have been triggered, so check invariants after all adds
@@ -236,13 +237,13 @@ public class TestIndexWriterMergePolicy
int segmentCount = writer.getSegmentCount();
for (int i = segmentCount - 1; i >= 0; i--) {
int docCount = writer.getDocCount(i);
- assertTrue(docCount > lowerBound);
+ assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " upperBound=" + upperBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString() + " config=" + writer.getConfig(), docCount > lowerBound);
if (docCount <= upperBound) {
numSegments++;
} else {
if (upperBound * mergeFactor <= maxMergeDocs) {
- assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString(), numSegments < mergeFactor);
+ assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString() + " config=" + writer.getConfig(), numSegments < mergeFactor);
}
do {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Mon May 9 13:19:28 2011
@@ -27,6 +27,8 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.DefaultSimilarityProvider;
+import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -46,7 +48,12 @@ public class TestMaxTermFrequency extend
dir = newDirectory();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(MockTokenizer.SIMPLE, true)).setMergePolicy(newInOrderLogMergePolicy());
- config.setSimilarityProvider(new TestSimilarity());
+ config.setSimilarityProvider(new DefaultSimilarityProvider() {
+ @Override
+ public Similarity get(String field) {
+ return new TestSimilarity();
+ }
+ });
RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
Document doc = new Document();
Field foo = newField("foo", "", Field.Store.NO, Field.Index.ANALYZED);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java Mon May 9 13:19:28 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.document.Field.
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -40,17 +41,22 @@ import org.apache.lucene.util.LuceneTest
*/
public class TestNorms extends LuceneTestCase {
- private class SimilarityOne extends DefaultSimilarity {
+ private class SimilarityProviderOne extends DefaultSimilarityProvider {
@Override
- public float computeNorm(FieldInvertState state) {
- // Disable length norm
- return state.getBoost();
- }
+ public Similarity get(String field) {
+ return new DefaultSimilarity() {
+ @Override
+ public float computeNorm(FieldInvertState state) {
+ // diable length norm
+ return state.getBoost();
+ }
+ };
+ }
}
private static final int NUM_FIELDS = 10;
- private SimilarityProvider similarityOne;
+ private SimilarityProvider similarityProviderOne;
private Analyzer anlzr;
private int numDocNorms;
private ArrayList<Float> norms;
@@ -61,7 +67,7 @@ public class TestNorms extends LuceneTes
@Override
public void setUp() throws Exception {
super.setUp();
- similarityOne = new SimilarityOne();
+ similarityProviderOne = new SimilarityProviderOne();
anlzr = new MockAnalyzer();
}
@@ -152,7 +158,7 @@ public class TestNorms extends LuceneTes
private void createIndex(Random random, Directory dir) throws IOException {
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(5).setSimilarityProvider(similarityOne).setMergePolicy(newInOrderLogMergePolicy()));
+ .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newInOrderLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
@@ -186,7 +192,7 @@ public class TestNorms extends LuceneTes
assertEquals("number of norms mismatches",numDocNorms,b.length);
ArrayList<Float> storedNorms = (i==1 ? modifiedNorms : norms);
for (int j = 0; j < b.length; j++) {
- float norm = similarityOne.get(field).decodeNormValue(b[j]);
+ float norm = similarityProviderOne.get(field).decodeNormValue(b[j]);
float norm1 = storedNorms.get(j).floatValue();
assertEquals("stored norm value of "+field+" for doc "+j+" is "+norm+" - a mismatch!", norm, norm1, 0.000001);
}
@@ -197,7 +203,7 @@ public class TestNorms extends LuceneTes
private void addDocs(Random random, Directory dir, int ndocs, boolean compound) throws IOException {
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5).setSimilarityProvider(similarityOne).setMergePolicy(newInOrderLogMergePolicy()));
+ .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newInOrderLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
@@ -222,7 +228,7 @@ public class TestNorms extends LuceneTes
// return unique norm values that are unchanged by encoding/decoding
private float nextNorm(String fname) {
float norm = lastNorm + normDelta;
- Similarity similarity = similarityOne.get(fname);
+ Similarity similarity = similarityProviderOne.get(fname);
do {
float norm1 = similarity.decodeNormValue(similarity.encodeNormValue(norm));
if (norm1 > lastNorm) {
@@ -261,7 +267,12 @@ public class TestNorms extends LuceneTes
public void testCustomEncoder() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
- config.setSimilarityProvider(new CustomNormEncodingSimilarity());
+ config.setSimilarityProvider(new DefaultSimilarityProvider() {
+ @Override
+ public Similarity get(String field) {
+ return new CustomNormEncodingSimilarity();
+ }
+ });
RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
Document doc = new Document();
Field foo = newField("foo", "", Field.Store.NO, Field.Index.ANALYZED);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Mon May 9 13:19:28 2011
@@ -35,27 +35,29 @@ import org.apache.lucene.search.Explanat
public class TestOmitTf extends LuceneTestCase {
- public static class SimpleSimilarity extends Similarity implements SimilarityProvider {
- @Override public float computeNorm(FieldInvertState state) { return state.getBoost(); }
- @Override public float tf(float freq) { return freq; }
- @Override public float sloppyFreq(int distance) { return 2.0f; }
- @Override public float idf(int docFreq, int numDocs) { return 1.0f; }
- @Override public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
- return new IDFExplanation() {
- @Override
- public float getIdf() {
- return 1.0f;
- }
- @Override
- public String explain() {
- return "Inexplicable";
- }
- };
- }
+ public static class SimpleSimilarityProvider implements SimilarityProvider {
public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
public float coord(int overlap, int maxOverlap) { return 1.0f; }
public Similarity get(String field) {
- return this;
+ return new Similarity() {
+
+ @Override public float computeNorm(FieldInvertState state) { return state.getBoost(); }
+ @Override public float tf(float freq) { return freq; }
+ @Override public float sloppyFreq(int distance) { return 2.0f; }
+ @Override public float idf(int docFreq, int numDocs) { return 1.0f; }
+ @Override public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
+ return new IDFExplanation() {
+ @Override
+ public float getIdf() {
+ return 1.0f;
+ }
+ @Override
+ public String explain() {
+ return "Inexplicable";
+ }
+ };
+ }
+ };
}
}
@@ -254,7 +256,7 @@ public class TestOmitTf extends LuceneTe
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).
setMaxBufferedDocs(2).
- setSimilarityProvider(new SimpleSimilarity()).
+ setSimilarityProvider(new SimpleSimilarityProvider()).
setMergePolicy(newInOrderLogMergePolicy(2))
);
writer.setInfoStream(VERBOSE ? System.out : null);
@@ -285,7 +287,7 @@ public class TestOmitTf extends LuceneTe
* Verify the index
*/
IndexSearcher searcher = new IndexSearcher(dir, true);
- searcher.setSimilarityProvider(new SimpleSimilarity());
+ searcher.setSimilarityProvider(new SimpleSimilarityProvider());
Term a = new Term("noTf", term);
Term b = new Term("tf", term);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java Mon May 9 13:19:28 2011
@@ -216,7 +216,7 @@ public class TestPerFieldCodecSupport ex
IndexFileNames.FIELD_INFOS_EXTENSION));
FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
assertEquals("faild for segment index: " + i, codec[i],
- codecInfo.codecs[fieldInfo.codecId]);
+ codecInfo.codecs[fieldInfo.getCodecId()]);
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Mon May 9 13:19:28 2011
@@ -78,9 +78,10 @@ public class TestSegmentMerger extends L
merger.add(reader2);
int docsMerged = merger.merge();
assertTrue(docsMerged == 2);
+ final FieldInfos fieldInfos = merger.fieldInfos();
//Should be able to open a new SegmentReader against the new directory
- SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, merger.fieldInfos().hasProx(),
- merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()),
+ SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, fieldInfos.hasProx(),
+ merger.getSegmentCodecs(), fieldInfos.hasVectors(), fieldInfos),
BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
assertTrue(mergedReader != null);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBoolean2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBoolean2.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBoolean2.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBoolean2.java Mon May 9 13:19:28 2011
@@ -210,7 +210,7 @@ public class TestBoolean2 extends Lucene
int[] expDocNrs = {2, 3};
SimilarityProvider oldSimilarity = searcher.getSimilarityProvider();
try {
- searcher.setSimilarityProvider(new DefaultSimilarity(){
+ searcher.setSimilarityProvider(new DefaultSimilarityProvider(){
@Override
public float coord(int overlap, int maxOverlap) {
return overlap / ((float)maxOverlap - 1);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java Mon May 9 13:19:28 2011
@@ -44,8 +44,8 @@ public class TestComplexExplanations ext
}
// must be static for weight serialization tests
- private static DefaultSimilarity createQnorm1Similarity() {
- return new DefaultSimilarity() {
+ private static DefaultSimilarityProvider createQnorm1Similarity() {
+ return new DefaultSimilarityProvider() {
@Override
public float queryNorm(float sumOfSquaredWeights) {
return 1.0f; // / (float) Math.sqrt(1.0f + sumOfSquaredWeights);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java Mon May 9 13:19:28 2011
@@ -97,7 +97,7 @@ public class TestConstantScoreQuery exte
searcher = newSearcher(reader);
// set a similarity that does not normalize our boost away
- searcher.setSimilarityProvider(new DefaultSimilarity() {
+ searcher.setSimilarityProvider(new DefaultSimilarityProvider() {
@Override
public float queryNorm(float sumOfSquaredWeights) {
return 1.0f;
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Mon May 9 13:19:28 2011
@@ -73,7 +73,12 @@ public class TestDisjunctionMaxQuery ext
}
}
- public SimilarityProvider sim = new TestSimilarity();
+ public SimilarityProvider sim = new DefaultSimilarityProvider() {
+ @Override
+ public Similarity get(String field) {
+ return new TestSimilarity();
+ }
+ };
public Directory index;
public IndexReader r;
public IndexSearcher s;
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Mon May 9 13:19:28 2011
@@ -296,25 +296,30 @@ public class TestMultiPhraseQuery extend
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
- searcher.setSimilarityProvider(new DefaultSimilarity() {
-
+ searcher.setSimilarityProvider(new DefaultSimilarityProvider() {
@Override
- public IDFExplanation idfExplain(Collection<Term> terms,
- IndexSearcher searcher) throws IOException {
- return new IDFExplanation() {
-
+ public Similarity get(String field) {
+ return new DefaultSimilarity() {
+
@Override
- public float getIdf() {
- return 10f;
- }
+ public IDFExplanation idfExplain(Collection<Term> terms,
+ IndexSearcher searcher) throws IOException {
+ return new IDFExplanation() {
- @Override
- public String explain() {
- return "just a test";
- }
-
+ @Override
+ public float getIdf() {
+ return 10f;
+ }
+
+ @Override
+ public String explain() {
+ return "just a test";
+ }
+
+ };
+ }
};
- }
+ }
});
MultiPhraseQuery query = new MultiPhraseQuery();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSimilarity.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSimilarity.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSimilarity.java Mon May 9 13:19:28 2011
@@ -39,27 +39,28 @@ import org.apache.lucene.search.Explanat
*/
public class TestSimilarity extends LuceneTestCase {
- public static class SimpleSimilarity extends Similarity implements SimilarityProvider {
- @Override public float computeNorm(FieldInvertState state) { return state.getBoost(); }
- @Override public float tf(float freq) { return freq; }
- @Override public float sloppyFreq(int distance) { return 2.0f; }
- @Override public float idf(int docFreq, int numDocs) { return 1.0f; }
- @Override public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
- return new IDFExplanation() {
- @Override
- public float getIdf() {
- return 1.0f;
- }
- @Override
- public String explain() {
- return "Inexplicable";
- }
- };
- }
+ public static class SimpleSimilarityProvider implements SimilarityProvider {
public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
public float coord(int overlap, int maxOverlap) { return 1.0f; }
public Similarity get(String field) {
- return this;
+ return new Similarity() {
+ @Override public float computeNorm(FieldInvertState state) { return state.getBoost(); }
+ @Override public float tf(float freq) { return freq; }
+ @Override public float sloppyFreq(int distance) { return 2.0f; }
+ @Override public float idf(int docFreq, int numDocs) { return 1.0f; }
+ @Override public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
+ return new IDFExplanation() {
+ @Override
+ public float getIdf() {
+ return 1.0f;
+ }
+ @Override
+ public String explain() {
+ return "Inexplicable";
+ }
+ };
+ }
+ };
}
}
@@ -67,7 +68,7 @@ public class TestSimilarity extends Luce
Directory store = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, store,
newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer())
- .setSimilarityProvider(new SimpleSimilarity()));
+ .setSimilarityProvider(new SimpleSimilarityProvider()));
Document d1 = new Document();
d1.add(newField("field", "a c", Field.Store.YES, Field.Index.ANALYZED));
@@ -81,7 +82,7 @@ public class TestSimilarity extends Luce
writer.close();
IndexSearcher searcher = newSearcher(reader);
- searcher.setSimilarityProvider(new SimpleSimilarity());
+ searcher.setSimilarityProvider(new SimpleSimilarityProvider());
Term a = new Term("field", "a");
Term b = new Term("field", "b");
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Mon May 9 13:19:28 2011
@@ -36,6 +36,8 @@ import org.apache.lucene.search.Explanat
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
@@ -50,7 +52,7 @@ public class TestPayloadNearQuery extend
private IndexSearcher searcher;
private IndexReader reader;
private Directory directory;
- private BoostingSimilarity similarity = new BoostingSimilarity();
+ private BoostingSimilarityProvider similarityProvider = new BoostingSimilarityProvider();
private byte[] payload2 = new byte[]{2};
private byte[] payload4 = new byte[]{4};
@@ -105,7 +107,7 @@ public class TestPayloadNearQuery extend
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer())
- .setSimilarityProvider(similarity));
+ .setSimilarityProvider(similarityProvider));
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
@@ -118,7 +120,7 @@ public class TestPayloadNearQuery extend
writer.close();
searcher = newSearcher(reader);
- searcher.setSimilarityProvider(similarity);
+ searcher.setSimilarityProvider(similarityProvider);
}
@Override
@@ -297,43 +299,57 @@ public class TestPayloadNearQuery extend
}
// must be static for weight serialization tests
- static class BoostingSimilarity extends DefaultSimilarity {
+ static class BoostingSimilarityProvider implements SimilarityProvider {
- @Override public float scorePayload(int docId, int start, int end, byte[] payload, int offset, int length) {
- //we know it is size 4 here, so ignore the offset/length
- return payload[offset];
- }
- //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- //Make everything else 1 so we see the effect of the payload
- //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- @Override public float computeNorm(FieldInvertState state) {
- return state.getBoost();
- }
-
- @Override public float queryNorm(float sumOfSquaredWeights) {
+ public float queryNorm(float sumOfSquaredWeights) {
return 1.0f;
}
-
- @Override public float sloppyFreq(int distance) {
+
+ public float coord(int overlap, int maxOverlap) {
return 1.0f;
}
+
+ public Similarity get(String field) {
+ return new DefaultSimilarity() {
+
+ @Override
+ public float scorePayload(int docId, int start, int end, byte[] payload, int offset, int length) {
+ //we know it is size 4 here, so ignore the offset/length
+ return payload[offset];
+ }
+
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ //Make everything else 1 so we see the effect of the payload
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ @Override
+ public float computeNorm(FieldInvertState state) {
+ return state.getBoost();
+ }
- @Override public float coord(int overlap, int maxOverlap) {
- return 1.0f;
- }
- @Override public float tf(float freq) {
- return 1.0f;
- }
- // idf used for phrase queries
- @Override public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
- return new IDFExplanation() {
- @Override
- public float getIdf() {
+ @Override
+ public float sloppyFreq(int distance) {
+ return 1.0f;
+ }
+
+ @Override
+ public float tf(float freq) {
return 1.0f;
}
- @Override
- public String explain() {
- return "Inexplicable";
+
+ // idf used for phrase queries
+ @Override
+ public IDFExplanation idfExplain(Collection<Term> terms, IndexSearcher searcher) throws IOException {
+ return new IDFExplanation() {
+ @Override
+ public float getIdf() {
+ return 1.0f;
+ }
+
+ @Override
+ public String explain() {
+ return "Inexplicable";
+ }
+ };
}
};
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Mon May 9 13:19:28 2011
@@ -18,8 +18,11 @@ package org.apache.lucene.search.payload
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.English;
+import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.CheckHits;
@@ -54,7 +57,7 @@ import java.io.IOException;
public class TestPayloadTermQuery extends LuceneTestCase {
private IndexSearcher searcher;
private IndexReader reader;
- private BoostingSimilarity similarity = new BoostingSimilarity();
+ private SimilarityProvider similarityProvider = new BoostingSimilarityProvider();
private byte[] payloadField = new byte[]{1};
private byte[] payloadMultiField1 = new byte[]{2};
private byte[] payloadMultiField2 = new byte[]{4};
@@ -110,7 +113,7 @@ public class TestPayloadTermQuery extend
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer())
- .setSimilarityProvider(similarity).setMergePolicy(newInOrderLogMergePolicy()));
+ .setSimilarityProvider(similarityProvider).setMergePolicy(newInOrderLogMergePolicy()));
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
@@ -125,7 +128,7 @@ public class TestPayloadTermQuery extend
writer.close();
searcher = newSearcher(reader);
- searcher.setSimilarityProvider(similarity);
+ searcher.setSimilarityProvider(similarityProvider);
}
@Override
@@ -220,7 +223,12 @@ public class TestPayloadTermQuery extend
new MaxPayloadFunction(), false);
IndexSearcher theSearcher = new IndexSearcher(directory, true);
- theSearcher.setSimilarityProvider(new FullSimilarity());
+ theSearcher.setSimilarityProvider(new DefaultSimilarityProvider() {
+ @Override
+ public Similarity get(String field) {
+ return new FullSimilarity();
+ }
+ });
TopDocs hits = searcher.search(query, null, 100);
assertTrue("hits is null and it shouldn't be", hits != null);
assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100);
@@ -283,46 +291,49 @@ public class TestPayloadTermQuery extend
}
// must be static for weight serialization tests
- static class BoostingSimilarity extends DefaultSimilarity {
-
- // TODO: Remove warning after API has been finalized
- @Override
- public float scorePayload(int docId, int start, int end, byte[] payload, int offset, int length) {
- //we know it is size 4 here, so ignore the offset/length
- return payload[offset];
- }
+ static class BoostingSimilarityProvider implements SimilarityProvider {
- //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- //Make everything else 1 so we see the effect of the payload
- //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- @Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost();
- }
-
- @Override
public float queryNorm(float sumOfSquaredWeights) {
return 1;
}
-
- @Override
- public float sloppyFreq(int distance) {
- return 1;
- }
-
- @Override
+
public float coord(int overlap, int maxOverlap) {
return 1;
}
- @Override
- public float idf(int docFreq, int numDocs) {
- return 1;
- }
+ public Similarity get(String field) {
+ return new DefaultSimilarity() {
+
+ // TODO: Remove warning after API has been finalized
+ @Override
+ public float scorePayload(int docId, int start, int end, byte[] payload, int offset, int length) {
+ //we know it is size 4 here, so ignore the offset/length
+ return payload[offset];
+ }
- @Override
- public float tf(float freq) {
- return freq == 0 ? 0 : 1;
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ //Make everything else 1 so we see the effect of the payload
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ @Override
+ public float computeNorm(FieldInvertState state) {
+ return state.getBoost();
+ }
+
+ @Override
+ public float sloppyFreq(int distance) {
+ return 1;
+ }
+
+ @Override
+ public float idf(int docFreq, int numDocs) {
+ return 1;
+ }
+
+ @Override
+ public float tf(float freq) {
+ return freq == 0 ? 0 : 1;
+ }
+ };
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Mon May 9 13:19:28 2011
@@ -37,7 +37,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TermQuery;
@@ -50,7 +50,7 @@ import org.apache.lucene.util.LuceneTest
public class TestPayloadSpans extends LuceneTestCase {
private IndexSearcher searcher;
- private SimilarityProvider similarity = new DefaultSimilarity();
+ private SimilarityProvider similarity = new DefaultSimilarityProvider();
protected IndexReader indexReader;
private IndexReader closeIndexReader;
private Directory directory;
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Mon May 9 13:19:28 2011
@@ -17,11 +17,13 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.IndexSearcher;
@@ -410,10 +412,14 @@ public class TestSpans extends LuceneTes
for (int i = 0; i < leaves.length; i++) {
- final SimilarityProvider sim = new DefaultSimilarity() {
- @Override
- public float sloppyFreq(int distance) {
- return 0.0f;
+ final SimilarityProvider sim = new DefaultSimilarityProvider() {
+ public Similarity get(String field) {
+ return new DefaultSimilarity() {
+ @Override
+ public float sloppyFreq(int distance) {
+ return 0.0f;
+ }
+ };
}
};
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestCollectionUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestCollectionUtil.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestCollectionUtil.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestCollectionUtil.java Mon May 9 13:19:28 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
public class TestCollectionUtil extends LuceneTestCase {
@@ -89,9 +90,30 @@ public class TestCollectionUtil extends
}
}
- // should produce no exceptions
- public void testEmptyArraySort() {
- List<Integer> list = Collections.emptyList();
+ public void testEmptyListSort() {
+ // should produce no exceptions
+ List<Integer> list = Arrays.asList(new Integer[0]);
+ CollectionUtil.quickSort(list);
+ CollectionUtil.mergeSort(list);
+ CollectionUtil.insertionSort(list);
+ CollectionUtil.quickSort(list, Collections.reverseOrder());
+ CollectionUtil.mergeSort(list, Collections.reverseOrder());
+ CollectionUtil.insertionSort(list, Collections.reverseOrder());
+
+ // check that empty non-random access lists pass sorting without ex (as sorting is not needed)
+ list = new LinkedList<Integer>();
+ CollectionUtil.quickSort(list);
+ CollectionUtil.mergeSort(list);
+ CollectionUtil.insertionSort(list);
+ CollectionUtil.quickSort(list, Collections.reverseOrder());
+ CollectionUtil.mergeSort(list, Collections.reverseOrder());
+ CollectionUtil.insertionSort(list, Collections.reverseOrder());
+ }
+
+ public void testOneElementListSort() {
+ // check that one-element non-random access lists pass sorting without ex (as sorting is not needed)
+ List<Integer> list = new LinkedList<Integer>();
+ list.add(1);
CollectionUtil.quickSort(list);
CollectionUtil.mergeSort(list);
CollectionUtil.insertionSort(list);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java Mon May 9 13:19:28 2011
@@ -191,7 +191,7 @@ public class TestFSTs extends LuceneTest
}
final char[] buffer = new char[end];
for (int i = 0; i < end; i++) {
- buffer[i] = (char) _TestUtil.nextInt(random, 97, 102);
+ buffer[i] = (char) _TestUtil.nextInt(r, 97, 102);
}
return new String(buffer, 0, end);
}
@@ -1122,6 +1122,9 @@ public class TestFSTs extends LuceneTest
System.exit(0);
}
+ if (dirOut == null)
+ return;
+
System.out.println(ord + " terms; " + fst.getNodeCount() + " nodes; " + fst.getArcCount() + " arcs; " + fst.getArcWithOutputCount() + " arcs w/ output; tot size " + fst.sizeInBytes());
if (fst.getNodeCount() < 100) {
Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
@@ -1138,7 +1141,7 @@ public class TestFSTs extends LuceneTest
System.out.println("Saved FST to fst.bin.");
if (!verify) {
- System.exit(0);
+ return;
}
System.out.println("\nNow verify...");
@@ -1183,41 +1186,54 @@ public class TestFSTs extends LuceneTest
// java -cp build/classes/test:build/classes/java:lib/junit-4.7.jar org.apache.lucene.util.automaton.fst.TestFSTs /x/tmp/allTerms3.txt out
public static void main(String[] args) throws IOException {
- final String wordsFileIn = args[0];
- final String dirOut = args[1];
- int idx = 2;
int prune = 0;
int limit = Integer.MAX_VALUE;
int inputMode = 0; // utf8
boolean storeOrds = false;
boolean storeDocFreqs = false;
boolean verify = true;
- while(idx < args.length) {
+
+ String wordsFileIn = null;
+ String dirOut = null;
+
+ int idx = 0;
+ while (idx < args.length) {
if (args[idx].equals("-prune")) {
- prune = Integer.valueOf(args[1+idx]);
+ prune = Integer.valueOf(args[1 + idx]);
idx++;
- }
- if (args[idx].equals("-limit")) {
- limit = Integer.valueOf(args[1+idx]);
+ } else if (args[idx].equals("-limit")) {
+ limit = Integer.valueOf(args[1 + idx]);
idx++;
- }
- if (args[idx].equals("-utf8")) {
+ } else if (args[idx].equals("-utf8")) {
inputMode = 0;
- }
- if (args[idx].equals("-utf32")) {
+ } else if (args[idx].equals("-utf32")) {
inputMode = 1;
- }
- if (args[idx].equals("-docFreq")) {
+ } else if (args[idx].equals("-docFreq")) {
storeDocFreqs = true;
- }
- if (args[idx].equals("-ords")) {
+ } else if (args[idx].equals("-ords")) {
storeOrds = true;
- }
- if (args[idx].equals("-noverify")) {
+ } else if (args[idx].equals("-noverify")) {
verify = false;
+ } else if (args[idx].startsWith("-")) {
+ System.err.println("Unrecognized option: " + args[idx]);
+ System.exit(-1);
+ } else {
+ if (wordsFileIn == null) {
+ wordsFileIn = args[idx];
+ } else if (dirOut == null) {
+ dirOut = args[idx];
+ } else {
+ System.err.println("Too many arguments, expected: input [output]");
+ System.exit(-1);
+ }
}
idx++;
}
+
+ if (wordsFileIn == null) {
+ System.err.println("No input file.");
+ System.exit(-1);
+ }
// ord benefits from share, docFreqs don't:
Modified: lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java?rev=1101016&r1=1101015&r2=1101016&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java (original)
+++ lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/collation/tokenattributes/CollatedTermAttributeImpl.java Mon May 9 13:19:28 2011
@@ -40,11 +40,12 @@ public class CollatedTermAttributeImpl e
}
@Override
- public int toBytesRef(BytesRef target) {
- target.bytes = collator.getCollationKey(toString()).toByteArray();
- target.offset = 0;
- target.length = target.bytes.length;
- return target.hashCode();
+ public int fillBytesRef() {
+ BytesRef bytes = getBytesRef();
+ bytes.bytes = collator.getCollationKey(toString()).toByteArray();
+ bytes.offset = 0;
+ bytes.length = bytes.bytes.length;
+ return bytes.hashCode();
}
}