You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/02/01 18:01:55 UTC
svn commit: r1239205 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
test-framework/java/org/apache/lucene/index/RandomCodec.java
test/org/apache/lucene/index/TestRollingUpdates.java
Author: mikemccand
Date: Wed Feb 1 17:01:55 2012
New Revision: 1239205
URL: http://svn.apache.org/viewvc?rev=1239205&view=rev
Log:
add doPackFST option (default to false) to MemoryCodec; randomize it during tests
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1239205&r1=1239204&r2=1239205&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Wed Feb 1 17:01:55 2012
@@ -81,9 +81,21 @@ import org.apache.lucene.util.fst.Util;
// the reality that it is actually written to disk, but
// loads itself in ram?
public class MemoryPostingsFormat extends PostingsFormat {
-
+
+ private final boolean doPackFST;
+
public MemoryPostingsFormat() {
+ this(false);
+ }
+
+ public MemoryPostingsFormat(boolean doPackFST) {
super("Memory");
+ this.doPackFST = doPackFST;
+ }
+
+ @Override
+ public String toString() {
+ return "PostingsFormat(name=" + getName() + " doPackFST= " + doPackFST + ")";
}
private static final boolean VERBOSE = false;
@@ -93,12 +105,14 @@ public class MemoryPostingsFormat extend
private final FieldInfo field;
private final Builder<BytesRef> builder;
private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+ private final boolean doPackFST;
private int termCount;
- public TermsWriter(IndexOutput out, FieldInfo field) {
+ public TermsWriter(IndexOutput out, FieldInfo field, boolean doPackFST) {
this.out = out;
this.field = field;
- builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, outputs);
+ this.doPackFST = doPackFST;
+ builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPackFST);
}
private class PostingsWriter extends PostingsConsumer {
@@ -230,7 +244,11 @@ public class MemoryPostingsFormat extend
}
out.writeVLong(sumDocFreq);
out.writeVInt(docCount);
- builder.finish().save(out);
+ FST<BytesRef> fst = builder.finish();
+ if (doPackFST) {
+ fst = fst.pack(3, Math.max(10, fst.getNodeCount()/4));
+ }
+ fst.save(out);
if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer());
}
}
@@ -256,7 +274,7 @@ public class MemoryPostingsFormat extend
throw new UnsupportedOperationException("this codec cannot index offsets");
}
if (VERBOSE) System.out.println("\naddField field=" + field.name);
- return new TermsWriter(out, field);
+ return new TermsWriter(out, field, doPackFST);
}
@Override
@@ -776,6 +794,9 @@ public class MemoryPostingsFormat extend
break;
}
final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
+ if (VERBOSE) {
+ System.out.println("load field=" + termsReader.field.name);
+ }
fields.put(termsReader.field.name, termsReader);
}
} finally {
Modified: lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java?rev=1239205&r1=1239204&r2=1239205&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java Wed Feb 1 17:01:55 2012
@@ -94,7 +94,7 @@ public class RandomCodec extends Lucene4
formats.add(new Lucene40WithOrds());
if (!useNoMemoryExpensiveCodec) {
formats.add(new SimpleTextPostingsFormat());
- formats.add(new MemoryPostingsFormat());
+ formats.add(new MemoryPostingsFormat(random.nextBoolean()));
}
Collections.shuffle(formats, random);
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1239205&r1=1239204&r2=1239205&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Wed Feb 1 17:01:55 2012
@@ -38,7 +38,7 @@ public class TestRollingUpdates extends
//provider.register(new MemoryCodec());
if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) {
- Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat()));
+ Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random.nextBoolean())));
}
final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));