You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/10/27 04:57:05 UTC
svn commit: r1189590 - in
/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index: ./
codecs/ codecs/lucene3x/ codecs/lucene40/ codecs/memory/ codecs/perfield/
codecs/pulsing/ codecs/sep/ codecs/simpletext/
Author: rmuir
Date: Thu Oct 27 02:57:03 2011
New Revision: 1189590
URL: http://svn.apache.org/viewvc?rev=1189590&view=rev
Log:
LUCENE-3490: remove PFCW from indexwriter, instead passing down Codec to documentswriter
Added:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java (with props)
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java (with props)
Removed:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldCodec.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/SegmentFormats.java
Modified:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java Thu Oct 27 02:57:03 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
import java.io.File;
@@ -40,7 +41,6 @@ import java.util.Map;
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.FSDirectory;
@@ -144,8 +144,8 @@ public class CheckIndex {
/** Name of the segment. */
public String name;
- /** CodecInfo used to read this segment. */
- public SegmentFormats codec;
+ /** Codec used to read this segment. */
+ public Codec codec;
/** Document count (does not take deletions into account). */
public int docCount;
@@ -490,9 +490,9 @@ public class CheckIndex {
SegmentReader reader = null;
try {
- final SegmentFormats fmt = info.getSegmentFormats();
- msg(" codec=" + fmt);
- segInfoStat.codec = fmt;
+ final Codec codec = info.getCodec();
+ msg(" codec=" + codec);
+ segInfoStat.codec = codec;
msg(" compound=" + info.getUseCompoundFile());
segInfoStat.compound = info.getUseCompoundFile();
msg(" hasProx=" + info.getHasProx());
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Thu Oct 27 02:57:03 2011
@@ -25,10 +25,9 @@ import java.util.HashSet;
import java.util.Map;
import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
-import org.apache.lucene.index.codecs.PostingsFormat;
+import org.apache.lucene.index.codecs.DocValuesFormat;
import org.apache.lucene.index.codecs.DocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
import org.apache.lucene.index.values.PerDocFieldValues;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils;
@@ -321,14 +320,12 @@ final class DocFieldProcessor extends Do
docValuesConsumerAndDocID.docID = docState.docID;
return docValuesConsumerAndDocID.docValuesConsumer;
}
- PerDocConsumer perDocConsumer = perDocConsumers.get(fieldInfo.getFormatId());
+ PerDocConsumer perDocConsumer = perDocConsumers.get(0);
if (perDocConsumer == null) {
- PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(fieldInfo.getFormatId());
- SegmentFormats formats = perDocWriteState.segmentFormats;
- assert formats.formats.length > fieldInfo.getFormatId();
- PostingsFormat codec = formats.formats[fieldInfo.getFormatId()];
- perDocConsumer = codec.docsConsumer(perDocWriteState);
- perDocConsumers.put(Integer.valueOf(fieldInfo.getFormatId()), perDocConsumer);
+ PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(0);
+ DocValuesFormat dvFormat = docState.docWriter.codec.docValuesFormat();
+ perDocConsumer = dvFormat.docsConsumer(perDocWriteState);
+ perDocConsumers.put(0, perDocConsumer);
}
boolean success = false;
DocValuesConsumer docValuesConsumer = null;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu Oct 27 02:57:03 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.index.Documents
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.AlreadyClosedException;
@@ -125,8 +126,11 @@ final class DocumentsWriter {
final DocumentsWriterPerThreadPool perThreadPool;
final FlushPolicy flushPolicy;
final DocumentsWriterFlushControl flushControl;
- DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers,
+
+ final Codec codec;
+ DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers,
BufferedDeletesStream bufferedDeletesStream) throws IOException {
+ this.codec = codec;
this.directory = directory;
this.indexWriter = writer;
this.similarityProvider = config.getSimilarityProvider();
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Thu Oct 27 02:57:03 2011
@@ -26,8 +26,7 @@ import java.text.NumberFormat;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
-import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
@@ -153,7 +152,7 @@ public class DocumentsWriterPerThread {
}
private final static boolean INFO_VERBOSE = false;
final DocumentsWriter parent;
- final CodecProvider codecProvider;
+ final Codec codec;
final IndexWriter writer;
final Directory directory;
final DocState docState;
@@ -184,7 +183,7 @@ public class DocumentsWriterPerThread {
this.fieldInfos = fieldInfos;
this.writer = parent.indexWriter;
this.infoStream = parent.infoStream;
- this.codecProvider = this.writer.codecs;
+ this.codec = parent.codec;
this.docState = new DocState(this);
this.docState.similarityProvider = parent.indexWriter.getConfig()
.getSimilarityProvider();
@@ -406,8 +405,8 @@ public class DocumentsWriterPerThread {
return numDocsInRAM;
}
- SegmentFormats getFormat() {
- return flushState.segmentFormats;
+ Codec getCodec() {
+ return flushState.codec;
}
/** Reset after a flush */
@@ -444,7 +443,7 @@ public class DocumentsWriterPerThread {
assert deleteSlice == null : "all deletes must be applied in prepareFlush";
flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos,
numDocsInRAM, writer.getConfig().getTermIndexInterval(),
- fieldInfos.buildSegmentFormats(true), pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
+ codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed())));
final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.;
// Apply delete-by-docID now (delete-byDocID only
// happens when an exception is hit processing that
@@ -475,12 +474,12 @@ public class DocumentsWriterPerThread {
try {
consumer.flush(flushState);
pendingDeletes.terms.clear();
- final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentFormats, fieldInfos.asReadOnly());
+ final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly());
if (infoStream != null) {
message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs");
message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
message("flushedFiles=" + newSegment.files());
- message("flushed codecs=" + newSegment.getSegmentFormats());
+ message("flushed codec=" + newSegment.getCodec());
}
flushedDocCount += flushState.numDocs;
@@ -557,9 +556,9 @@ public class DocumentsWriterPerThread {
bytesUsed.addAndGet(-(length *(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT)));
}
- PerDocWriteState newPerDocWriteState(int codecId) {
+ PerDocWriteState newPerDocWriteState(int formatId) {
assert segment != null;
- return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT);
+ return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, formatId, IOContext.DEFAULT);
}
void setInfoStream(PrintStream infoStream) {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Thu Oct 27 02:57:03 2011
@@ -20,8 +20,6 @@ import java.util.Iterator;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
-import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats.SegmentFormatsBuilder;
import org.apache.lucene.util.SetOnce;
/**
@@ -128,7 +126,6 @@ public abstract class DocumentsWriterPer
private final ThreadState[] perThreads;
private volatile int numThreadStatesActive;
- private CodecProvider codecProvider;
private FieldNumberBiMap globalFieldMap;
private final SetOnce<DocumentsWriter> documentsWriter = new SetOnce<DocumentsWriter>();
@@ -148,11 +145,9 @@ public abstract class DocumentsWriterPer
public void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) {
this.documentsWriter.set(documentsWriter); // thread pool is bound to DW
- final CodecProvider codecs = config.getCodecProvider();
- this.codecProvider = codecs;
this.globalFieldMap = globalFieldMap;
for (int i = 0; i < perThreads.length; i++) {
- final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentFormatsBuilder.create(codecs));
+ final FieldInfos infos = globalFieldMap.newFieldInfos();
perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain));
}
}
@@ -240,7 +235,7 @@ public abstract class DocumentsWriterPer
assert threadState.isHeldByCurrentThread();
final DocumentsWriterPerThread dwpt = threadState.perThread;
if (!closed) {
- final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentFormatsBuilder.create(codecProvider));
+ final FieldInfos infos = globalFieldMap.newFieldInfos();
final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos);
newDwpt.initialize();
threadState.resetWriter(newDwpt);
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfo.java Thu Oct 27 02:57:03 2011
@@ -21,7 +21,6 @@ import org.apache.lucene.index.values.Va
/** @lucene.experimental */
public final class FieldInfo {
- public static final int UNASSIGNED_FORMAT_ID = -1;
public final String name;
public final int number;
@@ -38,7 +37,6 @@ public final class FieldInfo {
public IndexOptions indexOptions;
public boolean storePayloads; // whether this field stores payloads together with term positions
- private int formatId = UNASSIGNED_FORMAT_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field
/**
* Controls how much information is stored in the postings lists.
@@ -77,21 +75,11 @@ public final class FieldInfo {
}
assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !storePayloads;
}
-
- public void setFormatId(int formatId) {
- assert this.formatId == UNASSIGNED_FORMAT_ID : "CodecId can only be set once.";
- this.formatId = formatId;
- }
-
- public int getFormatId() {
- return formatId;
- }
@Override
public Object clone() {
FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector,
storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
- clone.formatId = this.formatId;
return clone;
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FieldInfos.java Thu Oct 27 02:57:03 2011
@@ -29,9 +29,6 @@ import java.util.TreeMap;
import java.util.Map.Entry;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats.SegmentFormatsBuilder;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -152,8 +149,8 @@ public final class FieldInfos implements
* @return a new {@link FieldInfos} instance with this as the global field
* map
*/
- public FieldInfos newFieldInfos(SegmentFormatsBuilder segmentFormatsBuilder) {
- return new FieldInfos(this, segmentFormatsBuilder);
+ public FieldInfos newFieldInfos() {
+ return new FieldInfos(this);
}
/**
@@ -198,7 +195,6 @@ public final class FieldInfos implements
private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<Integer,FieldInfo>();
private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>();
private final FieldNumberBiMap globalFieldNumbers;
- private final SegmentFormatsBuilder segmentFormatsBuilder;
// First used in 2.9; prior to 2.9 there was no format header
public static final int FORMAT_START = -2;
@@ -230,16 +226,15 @@ public final class FieldInfos implements
/**
* Creates a new {@link FieldInfos} instance with a private
- * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder}
- * initialized with {@link CodecProvider#getDefault()}.
+ * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap}
* <p>
* Note: this ctor should not be used during indexing use
* {@link FieldInfos#FieldInfos(FieldInfos)} or
- * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.codecs.perfield.SegmentFormats.SegmentCodecsBuilder)}
+ * {@link FieldInfos#FieldInfos(FieldNumberBiMap)}
* instead.
*/
public FieldInfos() {
- this(new FieldNumberBiMap(), SegmentFormatsBuilder.create(CodecProvider.getDefault()));
+ this(new FieldNumberBiMap());
}
/**
@@ -249,7 +244,7 @@ public final class FieldInfos implements
* @see #isReadOnly()
*/
FieldInfos(FieldInfos other) {
- this(other.globalFieldNumbers, other.segmentFormatsBuilder);
+ this(other.globalFieldNumbers);
}
/**
@@ -257,9 +252,8 @@ public final class FieldInfos implements
* If the {@link FieldNumberBiMap} is <code>null</code> this instance will be read-only.
* @see #isReadOnly()
*/
- FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentFormatsBuilder segmentCodecsBuilder) {
+ FieldInfos(FieldNumberBiMap globalFieldNumbers) {
this.globalFieldNumbers = globalFieldNumbers;
- this.segmentFormatsBuilder = segmentCodecsBuilder;
}
/**
@@ -273,7 +267,7 @@ public final class FieldInfos implements
* @throws IOException
*/
public FieldInfos(Directory d, String name) throws IOException {
- this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only
+ this((FieldNumberBiMap)null); // use null here to make this FIs Read-Only
final IndexInput input = d.openInput(name, IOContext.READONCE);
try {
read(input, name);
@@ -309,7 +303,7 @@ public final class FieldInfos implements
*/
@Override
synchronized public Object clone() {
- FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentFormatsBuilder);
+ FieldInfos fis = new FieldInfos(globalFieldNumbers);
fis.format = format;
fis.hasFreq = hasFreq;
fis.hasProx = hasProx;
@@ -468,7 +462,6 @@ public final class FieldInfos implements
if (globalFieldNumbers == null) {
throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
}
- assert segmentFormatsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only";
FieldInfo fi = fieldInfo(name);
if (fi == null) {
final int fieldNumber = nextFieldNumber(name, preferredFieldNumber);
@@ -477,9 +470,6 @@ public final class FieldInfos implements
fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions);
fi.setDocValues(docValues);
}
- if ((fi.isIndexed || fi.hasDocValues()) && fi.getFormatId() == FieldInfo.UNASSIGNED_FORMAT_ID) {
- segmentFormatsBuilder.tryAddAndSet(fi);
- }
version++;
return fi;
}
@@ -569,22 +559,6 @@ public final class FieldInfos implements
}
return false;
}
-
- /**
- * Builds the {@link SegmentFormats} mapping for this {@link FieldInfos} instance.
- * @param clearBuilder <code>true</code> iff the internal {@link SegmentFormatsBuilder} must be cleared otherwise <code>false</code>
- */
- public SegmentFormats buildSegmentFormats(boolean clearBuilder) {
- if (globalFieldNumbers == null) {
- throw new IllegalStateException("FieldInfos are read-only no SegmentFormats available");
- }
- assert segmentFormatsBuilder != null;
- final SegmentFormats segmentFormats = segmentFormatsBuilder.build();
- if (clearBuilder) {
- segmentFormatsBuilder.clear();
- }
- return segmentFormats;
- }
public void write(Directory d, String name) throws IOException {
IndexOutput output = d.createOutput(name, IOContext.READONCE);
@@ -628,7 +602,6 @@ public final class FieldInfos implements
bits |= OMIT_POSITIONS;
output.writeString(fi.name);
output.writeInt(fi.number);
- output.writeInt(fi.getFormatId());
output.writeByte(bits);
final byte b;
@@ -698,9 +671,7 @@ public final class FieldInfos implements
for (int i = 0; i < size; i++) {
String name = input.readString();
- // if this is a previous format codec 0 will be preflex!
final int fieldNumber = format <= FORMAT_FLEX? input.readInt():i;
- final int formatId = format <= FORMAT_FLEX? input.readInt():0;
byte bits = input.readByte();
boolean isIndexed = (bits & IS_INDEXED) != 0;
boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
@@ -781,8 +752,7 @@ public final class FieldInfos implements
throw new IllegalStateException("unhandled indexValues type " + b);
}
}
- final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType);
- addInternal.setFormatId(formatId);
+ addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType);
}
if (input.getFilePointer() != input.length()) {
@@ -804,7 +774,7 @@ public final class FieldInfos implements
if (isReadOnly()) {
return this;
}
- final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null, null);
+ final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null);
for (FieldInfo fieldInfo : this) {
FieldInfo clone = (FieldInfo) (fieldInfo).clone();
roFis.putInternal(clone);
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Thu Oct 27 02:57:03 2011
@@ -57,7 +57,7 @@ final class FreqProxTermsWriter extends
// Sort by field name
CollectionUtil.quickSort(allFields);
- final FieldsConsumer consumer = state.segmentFormats.format().fieldsConsumer(state);
+ final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state);
boolean success = false;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu Oct 27 02:57:03 2011
@@ -40,8 +40,6 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor;
import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats.SegmentCodecsBuilder;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats.SegmentFormatsBuilder;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.CompoundFileDirectory;
@@ -907,7 +905,7 @@ public class IndexWriter implements Clos
// start with previous field numbers, but new FieldInfos
globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory);
- docWriter = new DocumentsWriter(config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
+ docWriter = new DocumentsWriter(codecs.getDefaultCodec(), config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
docWriter.setInfoStream(infoStream);
// Default deleter (for backwards compatibility) is
@@ -2568,7 +2566,7 @@ public class IndexWriter implements Clos
// abortable so that IW.close(false) is able to stop it
SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(),
mergedName, null, payloadProcessorProvider,
- globalFieldNumberMap.newFieldInfos(SegmentFormatsBuilder.create(codecs)), context);
+ globalFieldNumberMap.newFieldInfos(), codecs, context);
for (IndexReader reader : readers) // add new indexes
merger.add(reader);
@@ -2576,7 +2574,7 @@ public class IndexWriter implements Clos
final FieldInfos fieldInfos = merger.fieldInfos();
SegmentInfo info = new SegmentInfo(mergedName, docCount, directory,
- false, merger.getSegmentFormats(),
+ false, merger.getCodec(),
fieldInfos);
setDiagnostics(info, "addIndexes(IndexReader...)");
@@ -3460,7 +3458,7 @@ public class IndexWriter implements Clos
// Bind a new segment name here so even with
// ConcurrentMergePolicy we keep deterministic segment
// names.
- merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos(SegmentFormatsBuilder.create(codecs)));
+ merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos());
// Lock order: IW -> BD
final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments);
@@ -3634,7 +3632,7 @@ public class IndexWriter implements Clos
IOContext context = new IOContext(merge.getMergeInfo());
SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge,
- payloadProcessorProvider, merge.info.getFieldInfos(), context);
+ payloadProcessorProvider, merge.info.getFieldInfos(), codecs, context);
if (infoStream != null) {
message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors());
@@ -3680,10 +3678,10 @@ public class IndexWriter implements Clos
mergedDocCount = merge.info.docCount = merger.merge();
// Record which codec was used to write the segment
- merge.info.setSegmentFormats(merger.getSegmentFormats());
+ merge.info.setCodec(merger.getCodec());
if (infoStream != null) {
- message("merge segmentCodecs=" + merger.getSegmentFormats());
+ message("merge codecs=" + merger.getCodec());
message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size());
}
anyNonBulkMerges |= merger.getAnyNonBulkMerges();
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java Thu Oct 27 02:57:03 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
import java.io.PrintStream;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Counter;
@@ -35,7 +34,6 @@ public class PerDocWriteState {
public final String segmentName;
public final FieldInfos fieldInfos;
public final Counter bytesUsed;
- public final SegmentFormats segmentFormats;
public final int formatId;
public final IOContext context;
@@ -46,7 +44,6 @@ public class PerDocWriteState {
this.directory = directory;
this.segmentName = segmentName;
this.fieldInfos = fieldInfos;
- this.segmentFormats = fieldInfos.buildSegmentFormats(false);
this.formatId = codecId;
this.bytesUsed = bytesUsed;
this.context = context;
@@ -55,7 +52,6 @@ public class PerDocWriteState {
public PerDocWriteState(SegmentWriteState state) {
infoStream = state.infoStream;
directory = state.directory;
- segmentFormats = state.segmentFormats;
segmentName = state.segmentName;
fieldInfos = state.fieldInfos;
formatId = state.formatId;
@@ -68,7 +64,6 @@ public class PerDocWriteState {
this.directory = state.directory;
this.segmentName = state.segmentName;
this.fieldInfos = state.fieldInfos;
- this.segmentFormats = state.segmentFormats;
this.formatId = formatId;
this.bytesUsed = state.bytesUsed;
this.context = state.context;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Thu Oct 27 02:57:03 2011
@@ -20,11 +20,11 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.FieldsReader;
import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -69,7 +69,7 @@ final class SegmentCoreReaders {
}
segment = si.name;
- final SegmentFormats segmentFormats = si.getSegmentFormats();
+ final Codec codec = si.getCodec();
this.context = context;
this.dir = dir;
@@ -86,12 +86,12 @@ final class SegmentCoreReaders {
fieldInfos = si.getFieldInfos();
this.termsIndexDivisor = termsIndexDivisor;
- final PostingsFormat format = segmentFormats.format();
+ final PostingsFormat format = codec.postingsFormat();
final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor);
// Ask codec for its Fields
fields = format.fieldsProducer(segmentReadState);
assert fields != null;
- perDocProducer = format.docsProducer(segmentReadState);
+ perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
success = true;
} finally {
if (!success) {
@@ -166,8 +166,7 @@ final class SegmentCoreReaders {
}
final String storesSegment = si.getDocStoreSegment();
- // nocommit, shouldnt even see provider at this level
- fieldsReaderOrig = si.getSegmentFormats().provider.fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context,
+ fieldsReaderOrig = si.getCodec().fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context,
si.getDocStoreOffset(), si.docCount);
// Verify two sources of "maxDoc" agree:
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Thu Oct 27 02:57:03 2011
@@ -27,10 +27,9 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.lucene.index.codecs.PostingsFormat;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -98,7 +97,7 @@ public final class SegmentInfo implement
private FieldInfos fieldInfos;
- private SegmentFormats segmentFormats;
+ private Codec codec;
private Map<String,String> diagnostics;
@@ -117,7 +116,7 @@ public final class SegmentInfo implement
private long fieldInfosVersion;
public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile,
- SegmentFormats segmentFormats, FieldInfos fieldInfos) {
+ Codec codec, FieldInfos fieldInfos) {
this.name = name;
this.docCount = docCount;
this.dir = dir;
@@ -125,7 +124,7 @@ public final class SegmentInfo implement
this.isCompoundFile = isCompoundFile;
this.docStoreOffset = -1;
this.docStoreSegment = name;
- this.segmentFormats = segmentFormats;
+ this.codec = codec;
delCount = 0;
version = Constants.LUCENE_MAIN_VERSION;
this.fieldInfos = fieldInfos;
@@ -157,7 +156,7 @@ public final class SegmentInfo implement
}
isCompoundFile = src.isCompoundFile;
delCount = src.delCount;
- segmentFormats = src.segmentFormats;
+ codec = src.codec;
}
void setDiagnostics(Map<String, String> diagnostics) {
@@ -224,11 +223,10 @@ public final class SegmentInfo implement
// System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name);
if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
- segmentFormats = new SegmentFormats(codecs, input);
+ codec = codecs.lookup(input.readString());
} else {
- // codec ID on FieldInfo is 0 so it will simply use the first codec available
// TODO what todo if preflex is not available in the provider? register it or fail?
- segmentFormats = new SegmentFormats(codecs, new PostingsFormat[] { codecs.lookup("Lucene3x")});
+ codec = codecs.lookup("Lucene3x");
}
diagnostics = input.readStringStringMap();
@@ -351,7 +349,7 @@ public final class SegmentInfo implement
@Override
public Object clone() {
- final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, segmentFormats,
+ final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, codec,
fieldInfos == null ? null : (FieldInfos) fieldInfos.clone());
si.docStoreOffset = docStoreOffset;
si.docStoreSegment = docStoreSegment;
@@ -574,7 +572,7 @@ public final class SegmentInfo implement
output.writeByte((byte) (isCompoundFile ? YES : NO));
output.writeInt(delCount);
output.writeByte((byte) (hasProx));
- segmentFormats.write(output);
+ output.writeString(codec.getName());
output.writeStringStringMap(diagnostics);
output.writeByte((byte) (hasVectors));
}
@@ -584,16 +582,16 @@ public final class SegmentInfo implement
}
/** Can only be called once. */
- public void setSegmentFormats(SegmentFormats segmentFormats) {
- assert this.segmentFormats == null;
- if (segmentFormats == null) {
+ public void setCodec(Codec codec) {
+ assert this.codec == null;
+ if (codec == null) {
throw new IllegalArgumentException("segmentCodecs must be non-null");
}
- this.segmentFormats = segmentFormats;
+ this.codec = codec;
}
- SegmentFormats getSegmentFormats() {
- return segmentFormats;
+ Codec getCodec() {
+ return codec;
}
private void addIfExists(Set<String> files, String fileName) throws IOException {
@@ -629,7 +627,7 @@ public final class SegmentInfo implement
for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) {
addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext));
}
- segmentFormats.files(dir, this, fileSet);
+ codec.files(dir, this, fileSet);
}
if (docStoreOffset != -1) {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Thu Oct 27 02:57:03 2011
@@ -380,7 +380,7 @@ public final class SegmentInfos implemen
sis.cachedUnmodifiableList = null;
sis.cachedUnmodifiableSet = null;
for(final SegmentInfo info : this) {
- assert info.getSegmentFormats() != null;
+ assert info.getCodec() != null;
// dont directly access segments, use add method!!!
sis.add((SegmentInfo) info.clone());
}
@@ -1106,7 +1106,7 @@ public final class SegmentInfos implemen
if (cloneChildren) {
final List<SegmentInfo> list = new ArrayList<SegmentInfo>(size());
for(final SegmentInfo info : this) {
- assert info.getSegmentFormats() != null;
+ assert info.getCodec() != null;
list.add((SegmentInfo) info.clone());
}
return list;
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Thu Oct 27 02:57:03 2011
@@ -27,13 +27,14 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.MergePolicy.MergeAbortedException;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsReader;
import org.apache.lucene.index.codecs.FieldsWriter;
import org.apache.lucene.index.codecs.MergeState;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -67,14 +68,15 @@ final class SegmentMerger {
when merging stored fields */
private final static int MAX_RAW_MERGE_DOCS = 4192;
- private PostingsFormat format;
+ private final CodecProvider codecs;
+ private Codec codec;
private SegmentWriteState segmentWriteState;
private PayloadProcessorProvider payloadProcessorProvider;
private IOContext context;
- SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, IOContext context) {
+ SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, CodecProvider codecs, IOContext context) {
this.payloadProcessorProvider = payloadProcessorProvider;
directory = dir;
segment = name;
@@ -90,6 +92,7 @@ final class SegmentMerger {
};
}
this.termIndexInterval = termIndexInterval;
+ this.codecs = codecs;
this.context = context;
}
@@ -255,13 +258,12 @@ final class SegmentMerger {
fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false);
}
}
- final SegmentFormats codecInfo = fieldInfos.buildSegmentFormats(false);
+ codec = codecs.getDefaultCodec();
int docCount = 0;
setMatchingSegmentReaders();
- // nocommit
- final FieldsWriter fieldsWriter = codecInfo.provider.fieldsFormat().fieldsWriter(directory, segment, context);
+ final FieldsWriter fieldsWriter = codec.fieldsFormat().fieldsWriter(directory, segment, context);
try {
int idx = 0;
for (MergeState.IndexReaderAndLiveDocs reader : readers) {
@@ -295,7 +297,7 @@ final class SegmentMerger {
// entering the index. See LUCENE-1282 for
// details.
throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption");
- segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, context);
+ segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codec, null, context);
return docCount;
}
@@ -495,9 +497,9 @@ final class SegmentMerger {
}
}
- SegmentFormats getSegmentFormats() {
+ Codec getCodec() {
assert segmentWriteState != null;
- return segmentWriteState.segmentFormats;
+ return segmentWriteState.codec;
}
private final void mergeTerms() throws CorruptIndexException, IOException {
@@ -568,8 +570,8 @@ final class SegmentMerger {
mergeState.dirPayloadProcessor[i] = payloadProcessorProvider.getDirProcessor(reader.reader.directory());
}
}
- format = segmentWriteState.segmentFormats.format();
- final FieldsConsumer consumer = format.fieldsConsumer(segmentWriteState);
+
+ final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState);
boolean success = false;
try {
consumer.merge(mergeState,
@@ -586,7 +588,7 @@ final class SegmentMerger {
}
private void mergePerDoc() throws IOException {
- final PerDocConsumer docsConsumer = format
+ final PerDocConsumer docsConsumer = codec.docValuesFormat()
.docsConsumer(new PerDocWriteState(segmentWriteState));
// TODO: remove this check when 3.x indexes are no longer supported
// (3.x indexes don't have docvalues)
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java Thu Oct 27 02:57:03 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.index;
import java.io.PrintStream;
-import org.apache.lucene.index.codecs.perfield.SegmentFormats;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
@@ -44,7 +44,7 @@ public class SegmentWriteState {
// Lazily created:
public BitVector liveDocs;
- public final SegmentFormats segmentFormats;
+ public final Codec codec;
public final int formatId;
/** Expert: The fraction of terms in the "dictionary" which should be stored
@@ -57,7 +57,7 @@ public class SegmentWriteState {
public final IOContext context;
public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
- int numDocs, int termIndexInterval, SegmentFormats segmentFormats, BufferedDeletes segDeletes, IOContext context) {
+ int numDocs, int termIndexInterval, Codec codec, BufferedDeletes segDeletes, IOContext context) {
this.infoStream = infoStream;
this.segDeletes = segDeletes;
this.directory = directory;
@@ -65,7 +65,7 @@ public class SegmentWriteState {
this.fieldInfos = fieldInfos;
this.numDocs = numDocs;
this.termIndexInterval = termIndexInterval;
- this.segmentFormats = segmentFormats;
+ this.codec = codec;
formatId = -1;
this.context = context;
}
@@ -80,8 +80,8 @@ public class SegmentWriteState {
fieldInfos = state.fieldInfos;
numDocs = state.numDocs;
termIndexInterval = state.termIndexInterval;
- segmentFormats = state.segmentFormats;
context = state.context;
+ codec = state.codec;
this.formatId = formatId;
segDeletes = state.segDeletes;
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java Thu Oct 27 02:57:03 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.FieldsWriter;
import org.apache.lucene.store.IOContext;
@@ -35,12 +36,12 @@ final class StoredFieldsWriter {
int freeCount;
final DocumentsWriterPerThread.DocState docState;
- final CodecProvider codecProvider;
+ final Codec codec;
public StoredFieldsWriter(DocumentsWriterPerThread docWriter) {
this.docWriter = docWriter;
this.docState = docWriter.docState;
- this.codecProvider = docWriter.codecProvider;
+ this.codec = docWriter.codec;
}
private int numStoredFields;
@@ -81,8 +82,7 @@ final class StoredFieldsWriter {
private synchronized void initFieldsWriter(IOContext context) throws IOException {
if (fieldsWriter == null) {
- // nocommit
- fieldsWriter = codecProvider.getDefaultCodec().fieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context);
+ fieldsWriter = codec.fieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context);
lastDocID = 0;
}
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Thu Oct 27 02:57:03 2011
@@ -20,6 +20,9 @@ package org.apache.lucene.index.codecs;
import java.io.IOException;
import java.util.Set;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+
/**
* Encodes/decodes an inverted index segment
*/
@@ -36,11 +39,22 @@ public abstract class Codec {
public void getExtensions(Set<String> extensions) {
postingsFormat().getExtensions(extensions);
- fieldsFormat().getExtensions(extensions);
+ //TODO: not yet fieldsFormat().getExtensions(extensions);
+ docValuesFormat().getExtensions(extensions);
}
-
- /** Encodes/decodes postings and indexdocvalues */
+
+ public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ postingsFormat().files(dir, info, 0, files);
+ //TODO: not yet fieldsFormat().files(dir, info, files);
+ docValuesFormat().files(dir, info, 0, files);
+ }
+
+ /** Encodes/decodes postings */
public abstract PostingsFormat postingsFormat();
+
+ /** Encodes/decodes docvalues */
+ public abstract DocValuesFormat docValuesFormat();
+
/** Encodes/decodes stored fields, term vectors, fieldinfos */
public abstract FieldsFormat fieldsFormat();
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Thu Oct 27 02:57:03 2011
@@ -55,14 +55,14 @@ public class DefaultDocValuesConsumer ex
}
@SuppressWarnings("fallthrough")
- public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
+ public static void files(Directory dir, SegmentInfo segmentInfo, int formatId, Set<String> files) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
- if (fieldInfo.getFormatId() == codecId && fieldInfo.hasDocValues()) {
- files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
- files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ if (fieldInfo.hasDocValues()) {
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, formatId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, formatId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, formatId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, formatId, IndexFileNames.COMPOUND_FILE_EXTENSION));
return;
}
}
Added: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java?rev=1189590&view=auto
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java (added)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java Thu Oct 27 02:57:03 2011
@@ -0,0 +1,49 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.Directory;
+
+public class DefaultDocValuesFormat extends DocValuesFormat {
+
+ @Override
+ public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+ return new DefaultDocValuesConsumer(state);
+ }
+
+ @Override
+ public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+ return new DefaultDocValuesProducer(state);
+ }
+
+ @Override
+ public void getExtensions(Set<String> extensions) {
+ DefaultDocValuesConsumer.getExtensions(extensions);
+ }
+
+ @Override
+ public void files(Directory dir, SegmentInfo info, int formatId, Set<String> files) throws IOException {
+ DefaultDocValuesConsumer.files(dir, info, formatId, files);
+ }
+}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java Thu Oct 27 02:57:03 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -45,4 +46,9 @@ public class DefaultFieldsFormat extends
extensions.add(DefaultFieldsWriter.FIELDS_EXTENSION);
extensions.add(DefaultFieldsWriter.FIELDS_INDEX_EXTENSION);
}
+
+ @Override
+ public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
+ // TODO!
+ }
}
Added: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java?rev=1189590&view=auto
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java (added)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java Thu Oct 27 02:57:03 2011
@@ -0,0 +1,33 @@
+package org.apache.lucene.index.codecs;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.store.Directory;
+
+public abstract class DocValuesFormat {
+ public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
+ public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
+ public abstract void getExtensions(Set<String> extensions);
+ public abstract void files(Directory dir, SegmentInfo info, int formatId, Set<String> files) throws IOException;
+}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java Thu Oct 27 02:57:03 2011
@@ -72,7 +72,7 @@ public abstract class DocValuesReaderBas
try {
for (FieldInfo fieldInfo : fieldInfos) {
- if (codecId == fieldInfo.getFormatId() && fieldInfo.hasDocValues()) {
+ if (fieldInfo.hasDocValues()) {
final String field = fieldInfo.name;
// TODO can we have a compound file per segment and codec for
// docvalues?
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java Thu Oct 27 02:57:03 2011
@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -31,4 +32,5 @@ public abstract class FieldsFormat {
public abstract FieldsReader fieldsReader(Directory directory, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException;
public abstract FieldsWriter fieldsWriter(Directory directory, String segment, IOContext context) throws IOException;
public abstract void getExtensions(Set<String> extensions);
+ public abstract void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException;
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -55,10 +55,6 @@ public abstract class PostingsFormat {
* returns, it must hold open any files it will need to
* use; else, those files may be deleted. */
public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
-
- public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
-
- public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
/**
* Gathers files associated with this segment
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java Thu Oct 27 02:57:03 2011
@@ -17,10 +17,20 @@ package org.apache.lucene.index.codecs.l
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.PerDocWriteState;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.DefaultFieldsFormat;
+import org.apache.lucene.index.codecs.DocValuesFormat;
import org.apache.lucene.index.codecs.FieldsFormat;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsFormat;
+import org.apache.lucene.store.Directory;
/**
* Supports the Lucene 3.x index format (readonly)
@@ -31,13 +41,38 @@ public class Lucene3xCodec extends Codec
}
private final PostingsFormat postingsFormat = new Lucene3xPostingsFormat();
+
// TODO: this should really be a different impl
private final FieldsFormat fieldsFormat = new DefaultFieldsFormat();
+ // 3.x doesn't support docvalues
+ private final DocValuesFormat docValuesFormat = new DocValuesFormat() {
+ @Override
+ public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
+ return null;
+ }
+
+ @Override
+ public PerDocValues docsProducer(SegmentReadState state) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void getExtensions(Set<String> extensions) {}
+
+ @Override
+ public void files(Directory dir, SegmentInfo info, int formatId, Set<String> files) throws IOException {}
+ };
+
@Override
public PostingsFormat postingsFormat() {
return postingsFormat;
}
+
+ @Override
+ public DocValuesFormat docValuesFormat() {
+ return docValuesFormat;
+ }
@Override
public FieldsFormat fieldsFormat() {
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -22,14 +22,11 @@ import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.index.codecs.PostingsFormat;
-import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
/** Codec that reads the pre-flex-indexing postings
* format. It does not provide a writer because newly
@@ -81,14 +78,4 @@ public class Lucene3xPostingsFormat exte
extensions.add(TERMS_EXTENSION);
extensions.add(TERMS_INDEX_EXTENSION);
}
-
- @Override
- public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return null;
- }
-
- @Override
- public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return null;
- }
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.index.codecs.l
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -29,12 +28,8 @@ import org.apache.lucene.index.codecs.Po
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
import org.apache.lucene.index.codecs.BlockTreeTermsWriter;
import org.apache.lucene.index.codecs.PostingsFormat;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.Directory;
/** Default codec.
@@ -110,7 +105,6 @@ public class Lucene40PostingsFormat exte
public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
Lucene40PostingsReader.files(dir, segmentInfo, codecID, files);
BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
}
@Override
@@ -122,21 +116,10 @@ public class Lucene40PostingsFormat exte
extensions.add(FREQ_EXTENSION);
extensions.add(PROX_EXTENSION);
BlockTreeTermsReader.getExtensions(extensions);
- DefaultDocValuesConsumer.getExtensions(extensions);
}
@Override
public String toString() {
return name + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")";
}
-
- @Override
- public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state);
- }
-
- @Override
- public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state);
- }
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -797,22 +797,10 @@ public class MemoryPostingsFormat extend
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(EXTENSION);
- DefaultDocValuesConsumer.getExtensions(extensions);
- }
-
- @Override
- public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state);
- }
-
- @Override
- public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -49,20 +49,19 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.util.IOUtils;
/**
- * Enables per field format support. This class selects the format used to
- * write a field depending on the provided {@link SegmentFormats}. For each field
- * seen it resolves the format based on the {@link FieldInfo#formatId} which is
- * only valid during a segment merge. See {@link SegmentFormats} javadoc for
- * details.
+ * Enables per field format support.
*
- * @lucene.internal
+ * @lucene.experimental
*/
-final class PerFieldPostingsFormat extends PostingsFormat {
- private final SegmentFormats segmentFormats;
+// nocommit:
+// expose hook to lookup postings format by name
+// expose hook to get postingsformat for a field.
+// subclasses can deal with how they implement this (e.g. hashmap with default, solr schema, whatever)
+// this class can write its own private .per file with the mappings
+public abstract class PerFieldPostingsFormat extends PostingsFormat {
- PerFieldPostingsFormat(SegmentFormats segmentFormats) {
- super("PerField");
- this.segmentFormats = segmentFormats;
+ public PerFieldPostingsFormat(String name) {
+ super(name);
}
@Override
@@ -205,91 +204,4 @@ final class PerFieldPostingsFormat exten
format.getExtensions(extensions);
}
}
-
- @Override
- public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new PerDocConsumers(state);
- }
-
- @Override
- public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new PerDocProducers(state.dir, state.fieldInfos, state.segmentInfo,
- state.context, state.termsIndexDivisor);
- }
-
- private final class PerDocProducers extends PerDocValues {
- private final TreeMap<String, PerDocValues> codecs = new TreeMap<String, PerDocValues>();
-
- public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si,
- IOContext context, int indexDivisor) throws IOException {
- final Map<PostingsFormat, PerDocValues> producers = new HashMap<PostingsFormat, PerDocValues>();
- boolean success = false;
- try {
- for (FieldInfo fi : fieldInfos) {
- if (fi.hasDocValues()) {
- assert fi.getFormatId() != FieldInfo.UNASSIGNED_FORMAT_ID;
- PostingsFormat codec = segmentFormats.formats[fi.getFormatId()];
- if (!producers.containsKey(codec)) {
- producers.put(codec, codec.docsProducer(new SegmentReadState(dir,
- si, fieldInfos, context, indexDivisor, fi.getFormatId())));
- }
- codecs.put(fi.name, producers.get(codec));
- }
- }
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(producers.values());
- }
- }
- }
-
- @Override
- public Collection<String> fields() {
- return codecs.keySet();
- }
- @Override
- public IndexDocValues docValues(String field) throws IOException {
- final PerDocValues perDocProducer = codecs.get(field);
- if (perDocProducer == null) {
- return null;
- }
- return perDocProducer.docValues(field);
- }
-
- public void close() throws IOException {
- IOUtils.close(codecs.values());
- }
- }
-
- private final class PerDocConsumers extends PerDocConsumer {
- private final PerDocConsumer[] consumers;
- private final PostingsFormat[] formats;
- private final PerDocWriteState state;
-
- public PerDocConsumers(PerDocWriteState state) throws IOException {
- assert segmentFormats == state.segmentFormats;
- this.state = state;
- formats = segmentFormats.formats;
- consumers = new PerDocConsumer[formats.length];
- }
-
- public void close() throws IOException {
- IOUtils.close(consumers);
- }
-
- @Override
- public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
- final int formatId = field.getFormatId();
- assert formatId != FieldInfo.UNASSIGNED_FORMAT_ID;
- PerDocConsumer perDoc = consumers[formatId];
- if (perDoc == null) {
- perDoc = formats[formatId].docsConsumer(new PerDocWriteState(state, formatId));
- assert perDoc != null;
- consumers[formatId] = perDoc;
- }
- return perDoc.addValuesField(field);
- }
-
- }
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.index.codecs.p
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -29,23 +28,19 @@ import org.apache.lucene.index.codecs.Po
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
import org.apache.lucene.index.codecs.BlockTreeTermsWriter;
import org.apache.lucene.index.codecs.PostingsFormat;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat;
import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader;
import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter;
import org.apache.lucene.store.Directory;
-/** This codec "inlines" the postings for terms that have
- * low docFreq. It wraps another codec, which is used for
+/** This postings format "inlines" the postings for terms that have
+ * low docFreq. It wraps another postings format, which is used for
* writing the non-inlined terms.
*
* Currently in only inlines docFreq=1 terms, and
- * otherwise uses the normal "standard" codec.
+ * otherwise uses the normal "Lucene40" format.
* @lucene.experimental */
public class PulsingPostingsFormat extends PostingsFormat {
@@ -134,22 +129,10 @@ public class PulsingPostingsFormat exten
public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
Lucene40PostingsReader.files(dir, segmentInfo, codecID, files);
BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
}
@Override
public void getExtensions(Set<String> extensions) {
Lucene40PostingsFormat.getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getExtensions(extensions);
- }
-
- @Override
- public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state);
- }
-
- @Override
- public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java Thu Oct 27 02:57:03 2011
@@ -51,7 +51,7 @@ public class SepDocValuesConsumer extend
Set<String> files) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
- if (fieldInfo.getFormatId() == formatId && fieldInfo.hasDocValues()) {
+ if (fieldInfo.hasDocValues()) {
String filename = docValuesId(segmentInfo.name, formatId, fieldInfo.number);
switch (fieldInfo.getDocValues()) {
case BYTES_FIXED_DEREF:
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java?rev=1189590&r1=1189589&r2=1189590&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java Thu Oct 27 02:57:03 2011
@@ -20,18 +20,13 @@ package org.apache.lucene.index.codecs.s
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.codecs.PostingsFormat;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.Directory;
/** For debugging, curiosity, transparency only!! Do not
@@ -69,23 +64,10 @@ public class SimpleTextPostingsFormat ex
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name, id));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(POSTINGS_EXTENSION);
- DefaultDocValuesConsumer.getExtensions(extensions);
- }
-
- // TODO: would be great if these used a plain text impl
- @Override
- public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state);
- }
-
- @Override
- public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state);
}
}