You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/05/29 09:22:18 UTC
svn commit: r1343568 [7/10] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ lucene/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std31/
lucene/analysis/common/src/java/org/apache/lucene/analysis/stand...
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java Tue May 29 07:22:08 2012
@@ -32,7 +32,7 @@ import org.apache.lucene.util.MutableBit
class ReadersAndLiveDocs {
// Not final because we replace (clone) when we need to
// change it and it's been shared:
- public final SegmentInfo info;
+ public final SegmentInfoPerCommit info;
// Tracks how many consumers are using this instance:
private final AtomicInteger refCount = new AtomicInteger(1);
@@ -67,7 +67,7 @@ class ReadersAndLiveDocs {
// external NRT reader:
private boolean shared;
- public ReadersAndLiveDocs(IndexWriter writer, SegmentInfo info) {
+ public ReadersAndLiveDocs(IndexWriter writer, SegmentInfoPerCommit info) {
this.info = info;
this.writer = writer;
shared = true;
@@ -98,16 +98,16 @@ class ReadersAndLiveDocs {
int count;
if (liveDocs != null) {
count = 0;
- for(int docID=0;docID<info.docCount;docID++) {
+ for(int docID=0;docID<info.info.getDocCount();docID++) {
if (liveDocs.get(docID)) {
count++;
}
}
} else {
- count = info.docCount;
+ count = info.info.getDocCount();
}
- assert info.docCount - info.getDelCount() - pendingDeleteCount == count: "info.docCount=" + info.docCount + " info.getDelCount()=" + info.getDelCount() + " pendingDeleteCount=" + pendingDeleteCount + " count=" + count;;
+ assert info.info.getDocCount() - info.getDelCount() - pendingDeleteCount == count: "info.docCount=" + info.info.getDocCount() + " info.getDelCount()=" + info.getDelCount() + " pendingDeleteCount=" + pendingDeleteCount + " count=" + count;
return true;
}
@@ -169,7 +169,7 @@ class ReadersAndLiveDocs {
public synchronized boolean delete(int docID) {
assert liveDocs != null;
assert Thread.holdsLock(writer);
- assert docID >= 0 && docID < liveDocs.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + liveDocs.length() + " seg=" + info.name + " docCount=" + info.docCount;
+ assert docID >= 0 && docID < liveDocs.length() : "out of bounds: docid=" + docID + " liveDocsLength=" + liveDocs.length() + " seg=" + info.info.name + " docCount=" + info.info.getDocCount();
assert !shared;
final boolean didDelete = liveDocs.get(docID);
if (didDelete) {
@@ -207,7 +207,7 @@ class ReadersAndLiveDocs {
}
shared = true;
if (liveDocs != null) {
- return new SegmentReader(reader.getSegmentInfo(), reader.core, liveDocs, info.docCount - info.getDelCount() - pendingDeleteCount);
+ return new SegmentReader(reader.getSegmentInfo(), reader.core, liveDocs, info.info.getDocCount() - info.getDelCount() - pendingDeleteCount);
} else {
assert reader.getLiveDocs() == liveDocs;
reader.incRef();
@@ -217,17 +217,17 @@ class ReadersAndLiveDocs {
public synchronized void initWritableLiveDocs() throws IOException {
assert Thread.holdsLock(writer);
- assert info.docCount > 0;
+ assert info.info.getDocCount() > 0;
//System.out.println("initWritableLivedocs seg=" + info + " liveDocs=" + liveDocs + " shared=" + shared);
if (shared) {
// Copy on write: this means we've cloned a
// SegmentReader sharing the current liveDocs
// instance; must now make a private clone so we can
// change it:
- LiveDocsFormat liveDocsFormat = info.getCodec().liveDocsFormat();
+ LiveDocsFormat liveDocsFormat = info.info.getCodec().liveDocsFormat();
if (liveDocs == null) {
//System.out.println("create BV seg=" + info);
- liveDocs = liveDocsFormat.newLiveDocs(info.docCount);
+ liveDocs = liveDocsFormat.newLiveDocs(info.info.getDocCount());
} else {
liveDocs = liveDocsFormat.newLiveDocs(liveDocs);
}
@@ -270,25 +270,19 @@ class ReadersAndLiveDocs {
//System.out.println("rld.writeLiveDocs seg=" + info + " pendingDelCount=" + pendingDeleteCount);
if (pendingDeleteCount != 0) {
// We have new deletes
- assert liveDocs.length() == info.docCount;
-
- // Save in case we need to rollback on failure:
- final SegmentInfo sav = info.clone();
- info.advanceDelGen();
- info.setDelCount(info.getDelCount() + pendingDeleteCount);
+ assert liveDocs.length() == info.info.getDocCount();
// We can write directly to the actual name (vs to a
// .tmp & renaming it) because the file is not live
// until segments file is written:
- boolean success = false;
- try {
- info.getCodec().liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, dir, info, IOContext.DEFAULT);
- success = true;
- } finally {
- if (!success) {
- info.reset(sav);
- }
- }
+ info.info.getCodec().liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, dir, info, pendingDeleteCount, IOContext.DEFAULT);
+
+ // If we hit an exc in the line above (eg disk full)
+ // then info remains pointing to the previous
+ // (successfully written) del docs:
+ info.advanceDelGen();
+ info.setDelCount(info.getDelCount() + pendingDeleteCount);
+
pendingDeleteCount = 0;
return true;
} else {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Tue May 29 07:22:08 2012
@@ -80,30 +80,29 @@ final class SegmentCoreReaders {
private final Set<CoreClosedListener> coreClosedListeners =
Collections.synchronizedSet(new LinkedHashSet<CoreClosedListener>());
- SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, IOContext context, int termsIndexDivisor) throws IOException {
+ SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfoPerCommit si, IOContext context, int termsIndexDivisor) throws IOException {
if (termsIndexDivisor == 0) {
throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
}
- final Codec codec = si.getCodec();
+ final Codec codec = si.info.getCodec();
final Directory cfsDir; // confusing name: if (cfs) its the cfsdir, otherwise its the segment's directory.
boolean success = false;
try {
- if (si.getUseCompoundFile()) {
- cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
+ if (si.info.getUseCompoundFile()) {
+ cfsDir = cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
} else {
cfsReader = null;
cfsDir = dir;
}
- si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos
- fieldInfos = si.getFieldInfos();
-
+ fieldInfos = codec.fieldInfosFormat().getFieldInfosReader().read(cfsDir, si.info.name, IOContext.READONCE);
+
this.termsIndexDivisor = termsIndexDivisor;
final PostingsFormat format = codec.postingsFormat();
- final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor);
+ final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si.info, fieldInfos, context, termsIndexDivisor);
// Ask codec for its Fields
fields = format.fieldsProducer(segmentReadState);
assert fields != null;
@@ -113,10 +112,10 @@ final class SegmentCoreReaders {
norms = codec.normsFormat().docsProducer(segmentReadState);
perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
- fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(cfsDir, si, fieldInfos, context);
-
- if (si.getHasVectors()) { // open term vector files only as needed
- termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(cfsDir, si, fieldInfos, context);
+ fieldsReaderOrig = si.info.getCodec().storedFieldsFormat().fieldsReader(cfsDir, si.info, fieldInfos, context);
+
+ if (fieldInfos.hasVectors()) { // open term vector files only as needed
+ termVectorsReaderOrig = si.info.getCodec().termVectorsFormat().vectorsReader(cfsDir, si.info, fieldInfos, context);
} else {
termVectorsReaderOrig = null;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Tue May 29 07:22:08 2012
@@ -19,20 +19,16 @@ package org.apache.lucene.index;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.FieldInfosReader;
-import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.Constants;
+import org.apache.lucene.store.TrackingDirectoryWrapper;
/**
* Information about a segment such as it's name, directory, and files related
@@ -40,63 +36,25 @@ import org.apache.lucene.util.Constants;
*
* @lucene.experimental
*/
-public final class SegmentInfo implements Cloneable {
- // TODO: remove with hasVector and hasProx
- public static final int CHECK_FIELDINFO = -2;
+public final class SegmentInfo {
// TODO: remove these from this class, for now this is the representation
public static final int NO = -1; // e.g. no norms; no deletes;
public static final int YES = 1; // e.g. have norms; have deletes;
- public static final int WITHOUT_GEN = 0; // a file name that has no GEN in it.
- public String name; // unique name in dir
- public int docCount; // number of docs in seg
- public Directory dir; // where segment resides
-
- /*
- * Current generation of del file:
- * - NO if there are no deletes
- * - YES or higher if there are deletes at generation N
- */
- private long delGen;
-
- /*
- * Current generation of each field's norm file. If this array is null,
- * means no separate norms. If this array is not null, its values mean:
- * - NO says this field has no separate norms
- * >= YES says this field has separate norms with the specified generation
- */
- private Map<Integer,Long> normGen;
+ public final String name; // unique name in dir
+ private int docCount; // number of docs in seg
+ public final Directory dir; // where segment resides
private boolean isCompoundFile;
- private volatile List<String> files; // cached list of files that this segment uses
- // in the Directory
-
- private volatile long sizeInBytes = -1; // total byte size of all files (computed on demand)
-
- //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- private int docStoreOffset; // if this segment shares stored fields & vectors, this
- // offset is where in that file this segment's docs begin
- //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- private String docStoreSegment; // name used to derive fields/vectors file we share with
- // other segments
- //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- private boolean docStoreIsCompoundFile; // whether doc store files are stored in compound file (*.cfx)
-
- private int delCount; // How many deleted docs in this segment
-
- //TODO: remove when we don't have to support old indexes anymore that had this field
- private int hasVectors = CHECK_FIELDINFO;
- //TODO: remove when we don't have to support old indexes anymore that had this field
- private int hasProx = CHECK_FIELDINFO; // True if this segment has any fields with positional information
-
-
- private FieldInfos fieldInfos;
+ private volatile long sizeInBytes = -1; // total byte size of all files (computed on demand)
private Codec codec;
private Map<String,String> diagnostics;
+
+ private Map<String,String> attributes;
// Tracks the Lucene version this segment was created with, since 3.1. Null
// indicates an older than 3.0 index, and it's used to detect a too old index.
@@ -105,57 +63,6 @@ public final class SegmentInfo implement
// see Constants.LUCENE_MAIN_VERSION.
private String version;
- // NOTE: only used in-RAM by IW to track buffered deletes;
- // this is never written to/read from the Directory
- private long bufferedDeletesGen;
-
- // holds the fieldInfos Version to refresh files() cache if FI has changed
- private long fieldInfosVersion;
-
- public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile,
- Codec codec, FieldInfos fieldInfos) {
- this.name = name;
- this.docCount = docCount;
- this.dir = dir;
- delGen = NO;
- this.isCompoundFile = isCompoundFile;
- this.docStoreOffset = -1;
- this.docStoreSegment = name;
- this.codec = codec;
- delCount = 0;
- version = Constants.LUCENE_MAIN_VERSION;
- this.fieldInfos = fieldInfos;
- }
-
- /**
- * Copy everything from src SegmentInfo into our instance.
- */
- void reset(SegmentInfo src) {
- clearFilesCache();
- version = src.version;
- name = src.name;
- docCount = src.docCount;
- dir = src.dir;
- delGen = src.delGen;
- docStoreOffset = src.docStoreOffset;
- docStoreSegment = src.docStoreSegment;
- docStoreIsCompoundFile = src.docStoreIsCompoundFile;
- hasVectors = src.hasVectors;
- hasProx = src.hasProx;
- fieldInfos = src.fieldInfos == null ? null : src.fieldInfos.clone();
- if (src.normGen == null) {
- normGen = null;
- } else {
- normGen = new HashMap<Integer, Long>(src.normGen.size());
- for (Entry<Integer,Long> entry : src.normGen.entrySet()) {
- normGen.put(entry.getKey(), entry.getValue());
- }
- }
- isCompoundFile = src.isCompoundFile;
- delCount = src.delCount;
- codec = src.codec;
- }
-
void setDiagnostics(Map<String, String> diagnostics) {
this.diagnostics = diagnostics;
}
@@ -169,108 +76,34 @@ public final class SegmentInfo implement
* <p>Note: this is public only to allow access from
* the codecs package.</p>
*/
- public SegmentInfo(Directory dir, String version, String name, int docCount, long delGen, int docStoreOffset,
- String docStoreSegment, boolean docStoreIsCompoundFile, Map<Integer,Long> normGen, boolean isCompoundFile,
- int delCount, int hasProx, Codec codec, Map<String,String> diagnostics, int hasVectors) {
+ public SegmentInfo(Directory dir, String version, String name, int docCount,
+ boolean isCompoundFile, Codec codec, Map<String,String> diagnostics, Map<String,String> attributes) {
+ assert !(dir instanceof TrackingDirectoryWrapper);
this.dir = dir;
this.version = version;
this.name = name;
this.docCount = docCount;
- this.delGen = delGen;
- this.docStoreOffset = docStoreOffset;
- this.docStoreSegment = docStoreSegment;
- this.docStoreIsCompoundFile = docStoreIsCompoundFile;
- this.normGen = normGen;
this.isCompoundFile = isCompoundFile;
- this.delCount = delCount;
- this.hasProx = hasProx;
this.codec = codec;
this.diagnostics = diagnostics;
- this.hasVectors = hasVectors;
- }
-
- synchronized void loadFieldInfos(Directory dir, boolean checkCompoundFile) throws IOException {
- if (fieldInfos == null) {
- Directory dir0 = dir;
- if (isCompoundFile && checkCompoundFile) {
- dir0 = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(name,
- "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE, false);
- }
- try {
- FieldInfosReader reader = codec.fieldInfosFormat().getFieldInfosReader();
- fieldInfos = reader.read(dir0, name, IOContext.READONCE);
- } finally {
- if (dir != dir0) {
- dir0.close();
- }
- }
- }
+ this.attributes = attributes;
}
/**
- * Returns total size in bytes of all of files used by this segment
+ * Returns total size in bytes of all of files used by
+ * this segment. Note that this will not include any live
+ * docs for the segment; to include that use {@link
+ * SegmentInfoPerCommit#sizeInBytes()} instead.
*/
public long sizeInBytes() throws IOException {
+ if (sizeInBytes == -1) {
long sum = 0;
for (final String fileName : files()) {
sum += dir.fileLength(fileName);
}
sizeInBytes = sum;
- return sizeInBytes;
- }
-
- public boolean getHasVectors() throws IOException {
- return hasVectors == CHECK_FIELDINFO ? getFieldInfos().hasVectors() : hasVectors == YES;
- }
-
- public FieldInfos getFieldInfos() throws IOException {
- loadFieldInfos(dir, true);
- return fieldInfos;
- }
-
- public boolean hasDeletions() {
- // Cases:
- //
- // delGen == NO: this means this segment does not have deletions yet
- // delGen >= YES: this means this segment has deletions
- //
- return delGen != NO;
- }
-
- void advanceDelGen() {
- if (delGen == NO) {
- delGen = YES;
- } else {
- delGen++;
- }
- clearFilesCache();
- }
-
- void clearDelGen() {
- delGen = NO;
- clearFilesCache();
- }
-
- @Override
- public SegmentInfo clone() {
- final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, codec,
- fieldInfos == null ? null : fieldInfos.clone());
- si.docStoreOffset = docStoreOffset;
- si.docStoreSegment = docStoreSegment;
- si.docStoreIsCompoundFile = docStoreIsCompoundFile;
- si.delGen = delGen;
- si.delCount = delCount;
- si.diagnostics = new HashMap<String, String>(diagnostics);
- if (normGen != null) {
- si.normGen = new HashMap<Integer, Long>();
- for (Entry<Integer,Long> entry : normGen.entrySet()) {
- si.normGen.put(entry.getKey(), entry.getValue());
- }
}
- si.version = version;
- si.hasProx = hasProx;
- si.hasVectors = hasVectors;
- return si;
+ return sizeInBytes;
}
/**
@@ -278,17 +111,7 @@ public final class SegmentInfo implement
*/
@Deprecated
boolean hasSeparateNorms() {
- if (normGen == null) {
- return false;
- } else {
- for (long fieldNormGen : normGen.values()) {
- if (fieldNormGen >= YES) {
- return true;
- }
- }
- }
-
- return false;
+ return getAttribute(Lucene3xSegmentInfoFormat.NORMGEN_KEY) != null;
}
/**
@@ -299,7 +122,6 @@ public final class SegmentInfo implement
*/
void setUseCompoundFile(boolean isCompoundFile) {
this.isCompoundFile = isCompoundFile;
- clearFilesCache();
}
/**
@@ -310,87 +132,6 @@ public final class SegmentInfo implement
return isCompoundFile;
}
- public int getDelCount() {
- return delCount;
- }
-
- void setDelCount(int delCount) {
- this.delCount = delCount;
- assert delCount <= docCount;
- }
-
- /**
- * @deprecated shared doc stores are not supported in >= 4.0
- */
- @Deprecated
- public int getDocStoreOffset() {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- return docStoreOffset;
- }
-
- /**
- * @deprecated shared doc stores are not supported in >= 4.0
- */
- @Deprecated
- public boolean getDocStoreIsCompoundFile() {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- return docStoreIsCompoundFile;
- }
-
- /**
- * @deprecated shared doc stores are not supported in >= 4.0
- */
- @Deprecated
- public void setDocStoreIsCompoundFile(boolean docStoreIsCompoundFile) {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- this.docStoreIsCompoundFile = docStoreIsCompoundFile;
- clearFilesCache();
- }
-
- /**
- * @deprecated shared doc stores are not supported in >= 4.0
- */
- @Deprecated
- void setDocStore(int offset, String segment, boolean isCompoundFile) {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- docStoreOffset = offset;
- docStoreSegment = segment;
- docStoreIsCompoundFile = isCompoundFile;
- clearFilesCache();
- }
-
- /**
- * @deprecated shared doc stores are not supported in >= 4.0
- */
- @Deprecated
- public String getDocStoreSegment() {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- return docStoreSegment;
- }
-
- /**
- * @deprecated shared doc stores are not supported in >= 4.0
- */
- @Deprecated
- void setDocStoreOffset(int offset) {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- docStoreOffset = offset;
- clearFilesCache();
- }
-
- /**
- * @deprecated shared doc stores are not supported in 4.0
- */
- @Deprecated
- public void setDocStoreSegment(String docStoreSegment) {
- // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
- this.docStoreSegment = docStoreSegment;
- }
-
- public boolean getHasProx() throws IOException {
- return hasProx == CHECK_FIELDINFO ? getFieldInfos().hasProx() : hasProx == YES;
- }
-
/** Can only be called once. */
public void setCodec(Codec codec) {
assert this.codec == null;
@@ -404,34 +145,32 @@ public final class SegmentInfo implement
return codec;
}
+ public int getDocCount() {
+ if (this.docCount == -1) {
+ throw new IllegalStateException("docCount isn't set yet");
+ }
+ return docCount;
+ }
+
+ // NOTE: leave package private
+ void setDocCount(int docCount) {
+ if (this.docCount != -1) {
+ throw new IllegalStateException("docCount was already set");
+ }
+ this.docCount = docCount;
+ }
+
/*
* Return all files referenced by this SegmentInfo. The
* returns List is a locally cached List so you should not
* modify it.
*/
- public List<String> files() throws IOException {
- final long fisVersion = fieldInfosVersion;
- if (fisVersion != (fieldInfosVersion = getFieldInfos().getVersion())) {
- clearFilesCache(); // FIS has modifications - need to recompute
- } else if (files != null) {
- // Already cached:
- return files;
+ public Set<String> files() throws IOException {
+ if (setFiles == null) {
+ throw new IllegalStateException("files were not computed yet");
}
- final Set<String> fileSet = new HashSet<String>();
-
- codec.files(this, fileSet);
-
- files = new ArrayList<String>(fileSet);
-
- return files;
- }
-
- /* Called whenever any change is made that affects which
- * files this segment has. */
- private void clearFilesCache() {
- files = null;
- sizeInBytes = -1;
+ return Collections.unmodifiableSet(setFiles);
}
/** {@inheritDoc} */
@@ -443,16 +182,14 @@ public final class SegmentInfo implement
/** Used for debugging. Format may suddenly change.
*
* <p>Current format looks like
- * <code>_a(3.1):c45/4->_1</code>, which means the segment's
+ * <code>_a(3.1):c45/4</code>, which means the segment's
* name is <code>_a</code>; it was created with Lucene 3.1 (or
* '?' if it's unknown); it's using compound file
* format (would be <code>C</code> if not compound); it
* has 45 documents; it has 4 deletions (this part is
- * left off when there are no deletions); it's using the
- * shared doc stores named <code>_1</code> (this part is
- * left off if doc stores are private).</p>
+ * left off when there are no deletions).</p>
*/
- public String toString(Directory dir, int pendingDelCount) {
+ public String toString(Directory dir, int delCount) {
StringBuilder s = new StringBuilder();
s.append(name).append('(').append(version == null ? "?" : version).append(')').append(':');
@@ -462,35 +199,13 @@ public final class SegmentInfo implement
if (this.dir != dir) {
s.append('x');
}
- try {
- if (getHasVectors()) {
- s.append('v');
- }
- } catch (Throwable e) {
- // Messy: because getHasVectors may be used in an
- // thread-unsafe way, and may attempt to open an fnm
- // file that has since (legitimately) been deleted by
- // IndexWriter, instead of throwing these exceptions
- // up, just add v? to indicate we don't know if this
- // segment has vectors:
- s.append("v?");
- }
s.append(docCount);
- int delCount = getDelCount() + pendingDelCount;
if (delCount != 0) {
s.append('/').append(delCount);
}
- if (docStoreOffset != -1) {
- s.append("->").append(docStoreSegment);
- if (docStoreIsCompoundFile) {
- s.append('c');
- } else {
- s.append('C');
- }
- s.append('+').append(docStoreOffset);
- }
+ // TODO: we could append toString of attributes() here?
return s.toString();
}
@@ -533,34 +248,53 @@ public final class SegmentInfo implement
return version;
}
- long getBufferedDeletesGen() {
- return bufferedDeletesGen;
+ private Set<String> setFiles;
+
+ public void setFiles(Set<String> files) {
+ setFiles = files;
+ sizeInBytes = -1;
}
- void setBufferedDeletesGen(long v) {
- bufferedDeletesGen = v;
+ public void addFiles(Collection<String> files) {
+ setFiles.addAll(files);
}
-
- /** @lucene.internal */
- public long getDelGen() {
- return delGen;
+
+ public void addFile(String file) {
+ setFiles.add(file);
}
-
- /** @lucene.internal */
- public Map<Integer,Long> getNormGen() {
- return normGen;
+
+ /**
+ * Get a codec attribute value, or null if it does not exist
+ */
+ public String getAttribute(String key) {
+ if (attributes == null) {
+ return null;
+ } else {
+ return attributes.get(key);
+ }
}
- // TODO: clean up this SI/FI stuff here
- /** returns the 'real' value for hasProx (doesn't consult fieldinfos)
- * @lucene.internal */
- public int getHasProxInternal() {
- return hasProx;
+ /**
+ * Puts a codec attribute value.
+ * <p>
+ * This is a key-value mapping for the field that the codec can use
+ * to store additional metadata, and will be available to the codec
+ * when reading the segment via {@link #getAttribute(String)}
+ * <p>
+ * If a value already exists for the field, it will be replaced with
+ * the new value.
+ */
+ public String putAttribute(String key, String value) {
+ if (attributes == null) {
+ attributes = new HashMap<String,String>();
+ }
+ return attributes.put(key, value);
}
- /** returns the 'real' value for hasVectors (doesn't consult fieldinfos)
- * @lucene.internal */
- public int getHasVectorsInternal() {
- return hasVectors;
+ /**
+ * @return internal codec attributes map. May be null if no mappings exist.
+ */
+ public Map<String,String> attributes() {
+ return attributes;
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Tue May 29 07:22:08 2012
@@ -32,17 +32,21 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.SegmentInfosReader;
-import org.apache.lucene.codecs.SegmentInfosWriter;
-import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
+import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat;
+import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoReader;
import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexOutput;
+import org.apache.lucene.store.DataOutput; // javadocs
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.DataOutput; // javadocs
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NoSuchDirectoryException;
+import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ThreadInterruptedException;
/**
@@ -55,9 +59,8 @@ import org.apache.lucene.util.ThreadInte
* older segments_N files are present it's because they temporarily cannot be
* deleted, or, a writer is in the process of committing, or a custom
* {@link org.apache.lucene.index.IndexDeletionPolicy IndexDeletionPolicy}
- * is in use). This file lists each segment by name, has details about the
- * separate norms and deletion files, and also contains the size of each
- * segment.
+ * is in use). This file lists each segment by name and has details about the
+ * codec and generation of deletes.
* </p>
* <p>There is also a file <tt>segments.gen</tt>. This file contains
* the current generation (the <tt>_N</tt> in <tt>segments_N</tt>) of the index.
@@ -67,93 +70,84 @@ import org.apache.lucene.util.ThreadInte
* an {@link DataOutput#writeInt Int32} version header
* ({@link #FORMAT_SEGMENTS_GEN_CURRENT}), followed by the
* generation recorded as {@link DataOutput#writeLong Int64}, written twice.</p>
+ * <p>
+ * Files:
+ * <ul>
+ * <li><tt>segments.gen</tt>: GenHeader, Generation, Generation
+ * <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount,
+ * <SegName, SegCodec, DelGen, DeletionCount><sup>SegCount</sup>,
+ * CommitUserData, Checksum
+ * </ul>
+ * </p>
+ * Data types:
+ * <p>
+ * <ul>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>GenHeader, NameCounter, SegCount, DeletionCount --> {@link DataOutput#writeInt Int32}</li>
+ * <li>Generation, Version, DelGen, Checksum --> {@link DataOutput#writeLong Int64}</li>
+ * <li>SegName, SegCodec --> {@link DataOutput#writeString String}</li>
+ * <li>CommitUserData --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
+ * </ul>
+ * </p>
+ * Field Descriptions:
+ * <p>
+ * <ul>
+ * <li>Version counts how often the index has been changed by adding or deleting
+ * documents.</li>
+ * <li>NameCounter is used to generate names for new segment files.</li>
+ * <li>SegName is the name of the segment, and is used as the file name prefix for
+ * all of the files that compose the segment's index.</li>
+ * <li>DelGen is the generation count of the deletes file. If this is -1,
+ * there are no deletes. Anything above zero means there are deletes
+ * stored by {@link LiveDocsFormat}.</li>
+ * <li>DeletionCount records the number of deleted documents in this segment.</li>
+ * <li>Checksum contains the CRC32 checksum of all bytes in the segments_N file up
+ * until the checksum. This is used to verify integrity of the file on opening the
+ * index.</li>
+ * <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
+ * this segment.</li>
+ * <li>CommitUserData stores an optional user-supplied opaque
+ * Map<String,String> that was passed to {@link IndexWriter#commit(java.util.Map)}
+ * or {@link IndexWriter#prepareCommit(java.util.Map)}.</li>
+ * </ul>
+ * </p>
*
* @lucene.experimental
*/
-public final class SegmentInfos implements Cloneable, Iterable<SegmentInfo> {
+public final class SegmentInfos implements Cloneable, Iterable<SegmentInfoPerCommit> {
- /*
- * The file format version, a negative number.
- *
- * NOTE: future format numbers must always be one smaller
- * than the latest. With time, support for old formats will
- * be removed, however the numbers should continue to decrease.
+ /**
+ * The file format version for the segments_N codec header
*/
+ public static final int VERSION_40 = 0;
- // TODO: i don't think we need *all* these version numbers here?
- // most codecs only need FORMAT_CURRENT? and we should rename it
- // to FORMAT_FLEX? because the 'preamble' is just FORMAT_CURRENT + codecname
- // after that the codec takes over.
-
- // also i think this class should write this, somehow we let
- // preflexrw hackishly override this (like seek backwards and overwrite it)
-
- /** This format adds optional per-segment String
- * diagnostics storage, and switches userData to Map */
- public static final int FORMAT_DIAGNOSTICS = -9;
-
- /** Each segment records whether it has term vectors */
- public static final int FORMAT_HAS_VECTORS = -10;
-
- /** Each segment records the Lucene version that created it. */
- public static final int FORMAT_3_1 = -11;
-
- /** Each segment records whether its postings are written
- * in the new flex format */
- public static final int FORMAT_4_0 = -12;
-
- /** This must always point to the most recent file format.
- * whenever you add a new format, make it 1 smaller (negative version logic)! */
- // TODO: move this, as its currently part of required preamble
- public static final int FORMAT_CURRENT = FORMAT_4_0;
-
- /** This must always point to the first supported file format. */
- public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS;
-
/** Used for the segments.gen file only!
* Whenever you add a new format, make it 1 smaller (negative version logic)! */
public static final int FORMAT_SEGMENTS_GEN_CURRENT = -2;
-
+
public int counter; // used to name new segments
/**
* counts how often the index has been changed
*/
public long version;
-
+
private long generation; // generation of the "segments_N" for the next commit
private long lastGeneration; // generation of the "segments_N" file we last successfully read
- // or wrote; this is normally the same as generation except if
- // there was an IOException that had interrupted a commit
+ // or wrote; this is normally the same as generation except if
+ // there was an IOException that had interrupted a commit
public Map<String,String> userData = Collections.<String,String>emptyMap(); // Opaque Map<String, String> that user can specify during IndexWriter.commit
-
- private int format;
-
- private FieldNumberBiMap globalFieldNumberMap; // this segments global field number map - lazy loaded on demand
-
- private List<SegmentInfo> segments = new ArrayList<SegmentInfo>();
- private Set<SegmentInfo> segmentSet = new HashSet<SegmentInfo>();
- private transient List<SegmentInfo> cachedUnmodifiableList;
- private transient Set<SegmentInfo> cachedUnmodifiableSet;
- private Codec codecFormat;
+ private List<SegmentInfoPerCommit> segments = new ArrayList<SegmentInfoPerCommit>();
/**
* If non-null, information about loading segments_N files
* will be printed here. @see #setInfoStream.
*/
private static PrintStream infoStream = null;
-
- public void setFormat(int format) {
- this.format = format;
- }
-
- public int getFormat() {
- return format;
- }
- public SegmentInfo info(int i) {
+ public SegmentInfoPerCommit info(int i) {
return segments.get(i);
}
@@ -278,44 +272,50 @@ public final class SegmentInfos implemen
lastGeneration = generation;
- // TODO: scary to have default impl reopen the file... but to make it a bit more flexible,
- // maybe we could use a plain indexinput here... could default impl rewind/wrap with checksumII,
- // and any checksumming is then up to implementation?
- ChecksumIndexInput input = null;
+ ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ));
try {
- input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ));
final int format = input.readInt();
- setFormat(format);
-
- // check that it is a format we can understand
- if (format > FORMAT_MINIMUM)
- throw new IndexFormatTooOldException(input, format,
- FORMAT_MINIMUM, FORMAT_CURRENT);
- if (format < FORMAT_CURRENT)
- throw new IndexFormatTooNewException(input, format,
- FORMAT_MINIMUM, FORMAT_CURRENT);
-
- if (format <= FORMAT_4_0) {
- codecFormat = Codec.forName(input.readString());
+ if (format == CodecUtil.CODEC_MAGIC) {
+ // 4.0+
+ CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_40);
+ version = input.readLong();
+ counter = input.readInt();
+ int numSegments = input.readInt();
+ for(int seg=0;seg<numSegments;seg++) {
+ String segName = input.readString();
+ Codec codec = Codec.forName(input.readString());
+ //System.out.println("SIS.read seg=" + seg + " codec=" + codec);
+ SegmentInfo info = codec.segmentInfoFormat().getSegmentInfosReader().read(directory, segName, IOContext.READ);
+ info.setCodec(codec);
+ long delGen = input.readLong();
+ int delCount = input.readInt();
+ assert delCount <= info.getDocCount();
+ add(new SegmentInfoPerCommit(info, delCount, delGen));
+ }
+ userData = input.readStringStringMap();
} else {
- codecFormat = Codec.forName("Lucene3x");
+ Lucene3xSegmentInfoReader.readLegacyInfos(this, directory, input, format);
+ Codec codec = Codec.forName("Lucene3x");
+ for (SegmentInfoPerCommit info : this) {
+ info.info.setCodec(codec);
+ }
}
- SegmentInfosReader infosReader = codecFormat.segmentInfosFormat().getSegmentInfosReader();
- infosReader.read(directory, segmentFileName, input, this, IOContext.READ);
+
final long checksumNow = input.getChecksum();
final long checksumThen = input.readLong();
- if (checksumNow != checksumThen)
+ if (checksumNow != checksumThen) {
throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
- success = true;
- }
- finally {
- if (input != null) {
- input.close();
}
+
+ success = true;
+ } finally {
if (!success) {
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
this.clear();
+ IOUtils.closeWhileHandlingException(input);
+ } else {
+ input.close();
}
}
}
@@ -335,9 +335,9 @@ public final class SegmentInfos implemen
// Only non-null after prepareCommit has been called and
// before finishCommit is called
- IndexOutput pendingSegnOutput;
+ ChecksumIndexOutput pendingSegnOutput;
- private void write(Directory directory, Codec codec) throws IOException {
+ private void write(Directory directory) throws IOException {
String segmentFileName = getNextSegmentFileName();
@@ -348,15 +348,38 @@ public final class SegmentInfos implemen
generation++;
}
- IndexOutput segnOutput = null;
-
-
+ ChecksumIndexOutput segnOutput = null;
boolean success = false;
+ final Set<String> upgradedSIFiles = new HashSet<String>();
+
try {
- SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter();
- segnOutput = infosWriter.writeInfos(directory, segmentFileName, codec.getName(), this, IOContext.DEFAULT);
- infosWriter.prepareCommit(segnOutput);
+ segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName, IOContext.DEFAULT));
+ CodecUtil.writeHeader(segnOutput, "segments", VERSION_40);
+ segnOutput.writeLong(version);
+ segnOutput.writeInt(counter); // write counter
+ segnOutput.writeInt(size()); // write infos
+ for (SegmentInfoPerCommit siPerCommit : this) {
+ SegmentInfo si = siPerCommit.info;
+ segnOutput.writeString(si.name);
+ segnOutput.writeString(si.getCodec().getName());
+ segnOutput.writeLong(siPerCommit.getDelGen());
+ segnOutput.writeInt(siPerCommit.getDelCount());
+ assert si.dir == directory;
+
+ assert siPerCommit.getDelCount() <= si.getDocCount();
+
+ // If this segment is pre-4.x, perform a one-time
+ // "ugprade" to write the .si file for it:
+ String version = si.getVersion();
+ if (version == null || StringHelper.getVersionComparator().compare(version, "4.0") < 0) {
+ String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfoFormat.UPGRADED_SI_EXTENSION);
+ if (!directory.fileExists(fileName)) {
+ upgradedSIFiles.add(write3xInfo(directory, si, IOContext.DEFAULT));
+ }
+ }
+ }
+ segnOutput.writeStringStringMap(userData);
pendingSegnOutput = segnOutput;
success = true;
} finally {
@@ -364,6 +387,15 @@ public final class SegmentInfos implemen
// We hit an exception above; try to close the file
// but suppress any exception:
IOUtils.closeWhileHandlingException(segnOutput);
+
+ for(String fileName : upgradedSIFiles) {
+ try {
+ directory.deleteFile(fileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
+ }
+
try {
// Try not to leave a truncated segments_N file in
// the index:
@@ -375,17 +407,47 @@ public final class SegmentInfos implemen
}
}
- /** Prunes any segment whose docs are all deleted. */
- public void pruneDeletedSegments() {
- for(final Iterator<SegmentInfo> it = segments.iterator(); it.hasNext();) {
- final SegmentInfo info = it.next();
- if (info.getDelCount() == info.docCount) {
- it.remove();
- final boolean didRemove = segmentSet.remove(info);
- assert didRemove;
+ @Deprecated
+ public static String write3xInfo(Directory dir, SegmentInfo si, IOContext context) throws IOException {
+
+ // NOTE: this is NOT how 3.x is really written...
+ String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene3xSegmentInfoFormat.UPGRADED_SI_EXTENSION);
+ si.addFile(fileName);
+
+ //System.out.println("UPGRADE write " + fileName);
+ boolean success = false;
+ IndexOutput output = dir.createOutput(fileName, context);
+ try {
+ // we are about to write this SI in 3.x format, dropping all codec information, etc.
+ // so it had better be a 3.x segment or you will get very confusing errors later.
+ assert si.getCodec() instanceof Lucene3xCodec : "broken test, trying to mix preflex with other codecs";
+ CodecUtil.writeHeader(output, Lucene3xSegmentInfoFormat.UPGRADED_SI_CODEC_NAME,
+ Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_CURRENT);
+ // Write the Lucene version that created this segment, since 3.1
+ output.writeString(si.getVersion());
+ output.writeInt(si.getDocCount());
+
+ output.writeStringStringMap(si.attributes());
+
+ output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
+ output.writeStringStringMap(si.getDiagnostics());
+ output.writeStringSet(si.files());
+
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(output);
+ try {
+ si.dir.deleteFile(fileName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
+ } else {
+ output.close();
}
}
- assert segmentSet.size() == segments.size();
+
+ return fileName;
}
/**
@@ -398,12 +460,9 @@ public final class SegmentInfos implemen
try {
final SegmentInfos sis = (SegmentInfos) super.clone();
// deep clone, first recreate all collections:
- sis.segments = new ArrayList<SegmentInfo>(size());
- sis.segmentSet = new HashSet<SegmentInfo>(size());
- sis.cachedUnmodifiableList = null;
- sis.cachedUnmodifiableSet = null;
- for(final SegmentInfo info : this) {
- assert info.getCodec() != null;
+ sis.segments = new ArrayList<SegmentInfoPerCommit>(size());
+ for(final SegmentInfoPerCommit info : this) {
+ assert info.info.getCodec() != null;
// dont directly access segments, use add method!!!
sis.add(info.clone());
}
@@ -746,10 +805,11 @@ public final class SegmentInfos implemen
* method if changes have been made to this {@link SegmentInfos} instance
* </p>
**/
- final void prepareCommit(Directory dir, Codec codec) throws IOException {
- if (pendingSegnOutput != null)
+ final void prepareCommit(Directory dir) throws IOException {
+ if (pendingSegnOutput != null) {
throw new IllegalStateException("prepareCommit was already called");
- write(dir, codec);
+ }
+ write(dir);
}
/** Returns all file names referenced by SegmentInfo
@@ -771,26 +831,31 @@ public final class SegmentInfos implemen
}
final int size = size();
for(int i=0;i<size;i++) {
- final SegmentInfo info = info(i);
- if (info.dir == dir) {
- files.addAll(info(i).files());
+ final SegmentInfoPerCommit info = info(i);
+ assert info.info.dir == dir;
+ if (info.info.dir == dir) {
+ files.addAll(info.files());
}
}
return files;
}
- final void finishCommit(Directory dir, Codec codec) throws IOException {
- if (pendingSegnOutput == null)
+ final void finishCommit(Directory dir) throws IOException {
+ if (pendingSegnOutput == null) {
throw new IllegalStateException("prepareCommit was not called");
+ }
boolean success = false;
try {
- SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter();
- infosWriter.finishCommit(pendingSegnOutput);
- pendingSegnOutput = null;
+ pendingSegnOutput.finishCommit();
success = true;
} finally {
- if (!success)
+ if (!success) {
+ IOUtils.closeWhileHandlingException(pendingSegnOutput);
rollbackCommit(dir);
+ } else {
+ pendingSegnOutput.close();
+ pendingSegnOutput = null;
+ }
}
// NOTE: if we crash here, we have left a segments_N
@@ -852,9 +917,9 @@ public final class SegmentInfos implemen
* method if changes have been made to this {@link SegmentInfos} instance
* </p>
**/
- final void commit(Directory dir, Codec codec) throws IOException {
- prepareCommit(dir, codec);
- finishCommit(dir, codec);
+ final void commit(Directory dir) throws IOException {
+ prepareCommit(dir);
+ finishCommit(dir);
}
public String toString(Directory directory) {
@@ -865,7 +930,7 @@ public final class SegmentInfos implemen
if (i > 0) {
buffer.append(' ');
}
- final SegmentInfo info = info(i);
+ final SegmentInfoPerCommit info = info(i);
buffer.append(info.toString(directory, 0));
}
return buffer.toString();
@@ -890,15 +955,14 @@ public final class SegmentInfos implemen
void replace(SegmentInfos other) {
rollbackSegmentInfos(other.asList());
lastGeneration = other.lastGeneration;
- format = other.format;
}
/** Returns sum of all segment's docCounts. Note that
* this does not include deletions */
public int totalDocCount() {
int count = 0;
- for(SegmentInfo info : this) {
- count += info.docCount;
+ for(SegmentInfoPerCommit info : this) {
+ count += info.info.getDocCount();
}
return count;
}
@@ -909,36 +973,14 @@ public final class SegmentInfos implemen
version++;
}
- /**
- * Loads or returns the already loaded the global field number map for this {@link SegmentInfos}.
- * If this {@link SegmentInfos} has no global field number map the returned instance is empty
- */
- FieldNumberBiMap getOrLoadGlobalFieldNumberMap() throws IOException {
- if (globalFieldNumberMap != null) {
- return globalFieldNumberMap;
- }
- final FieldNumberBiMap map = new FieldNumberBiMap();
-
- if (size() > 0) {
- // build the map up
- for (SegmentInfo info : this) {
- final FieldInfos segFieldInfos = info.getFieldInfos();
- for (FieldInfo fi : segFieldInfos) {
- map.addOrGet(fi.name, fi.number);
- }
- }
- }
- return globalFieldNumberMap = map;
- }
-
/** applies all changes caused by committing a merge to this SegmentInfos */
void applyMergeChanges(MergePolicy.OneMerge merge, boolean dropSegment) {
- final Set<SegmentInfo> mergedAway = new HashSet<SegmentInfo>(merge.segments);
+ final Set<SegmentInfoPerCommit> mergedAway = new HashSet<SegmentInfoPerCommit>(merge.segments);
boolean inserted = false;
int newSegIdx = 0;
for (int segIdx = 0, cnt = segments.size(); segIdx < cnt; segIdx++) {
assert segIdx >= newSegIdx;
- final SegmentInfo info = segments.get(segIdx);
+ final SegmentInfoPerCommit info = segments.get(segIdx);
if (mergedAway.contains(info)) {
if (!inserted && !dropSegment) {
segments.set(segIdx, merge.info);
@@ -962,110 +1004,68 @@ public final class SegmentInfos implemen
if (!inserted && !dropSegment) {
segments.add(0, merge.info);
}
-
- // update the Set
- if (!dropSegment) {
- segmentSet.add(merge.info);
- }
- segmentSet.removeAll(mergedAway);
-
- assert segmentSet.size() == segments.size();
}
- List<SegmentInfo> createBackupSegmentInfos(boolean cloneChildren) {
- if (cloneChildren) {
- final List<SegmentInfo> list = new ArrayList<SegmentInfo>(size());
- for(final SegmentInfo info : this) {
- assert info.getCodec() != null;
- list.add(info.clone());
- }
- return list;
- } else {
- return new ArrayList<SegmentInfo>(segments);
+ List<SegmentInfoPerCommit> createBackupSegmentInfos() {
+ final List<SegmentInfoPerCommit> list = new ArrayList<SegmentInfoPerCommit>(size());
+ for(final SegmentInfoPerCommit info : this) {
+ assert info.info.getCodec() != null;
+ list.add(info.clone());
}
+ return list;
}
- void rollbackSegmentInfos(List<SegmentInfo> infos) {
+ void rollbackSegmentInfos(List<SegmentInfoPerCommit> infos) {
this.clear();
this.addAll(infos);
}
- /**
- * Returns the codec used to decode this SegmentInfos from disk
- * @lucene.internal
- */
- Codec codecFormat() {
- return codecFormat;
- }
-
/** Returns an <b>unmodifiable</b> {@link Iterator} of contained segments in order. */
// @Override (comment out until Java 6)
- public Iterator<SegmentInfo> iterator() {
+ public Iterator<SegmentInfoPerCommit> iterator() {
return asList().iterator();
}
/** Returns all contained segments as an <b>unmodifiable</b> {@link List} view. */
- public List<SegmentInfo> asList() {
- if (cachedUnmodifiableList == null) {
- cachedUnmodifiableList = Collections.unmodifiableList(segments);
- }
- return cachedUnmodifiableList;
- }
-
- /** Returns all contained segments as an <b>unmodifiable</b> {@link Set} view.
- * The iterator is not sorted, use {@link List} view or {@link #iterator} to get all segments in order. */
- public Set<SegmentInfo> asSet() {
- if (cachedUnmodifiableSet == null) {
- cachedUnmodifiableSet = Collections.unmodifiableSet(segmentSet);
- }
- return cachedUnmodifiableSet;
+ public List<SegmentInfoPerCommit> asList() {
+ return Collections.unmodifiableList(segments);
}
public int size() {
return segments.size();
}
- public void add(SegmentInfo si) {
- if (segmentSet.contains(si)) {
- throw new IllegalStateException("Cannot add the same segment two times to this SegmentInfos instance");
- }
+ public void add(SegmentInfoPerCommit si) {
segments.add(si);
- segmentSet.add(si);
- assert segmentSet.size() == segments.size();
}
- public void addAll(Iterable<SegmentInfo> sis) {
- for (final SegmentInfo si : sis) {
+ public void addAll(Iterable<SegmentInfoPerCommit> sis) {
+ for (final SegmentInfoPerCommit si : sis) {
this.add(si);
}
}
public void clear() {
segments.clear();
- segmentSet.clear();
}
-
- public void remove(SegmentInfo si) {
- final int index = this.indexOf(si);
- if (index >= 0) {
- this.remove(index);
- }
+
+ /** WARNING: O(N) cost */
+ public void remove(SegmentInfoPerCommit si) {
+ segments.remove(si);
}
- public void remove(int index) {
- segmentSet.remove(segments.remove(index));
- assert segmentSet.size() == segments.size();
+ /** WARNING: O(N) cost */
+ void remove(int index) {
+ segments.remove(index);
}
-
- public boolean contains(SegmentInfo si) {
- return segmentSet.contains(si);
+
+ /** WARNING: O(N) cost */
+ boolean contains(SegmentInfoPerCommit si) {
+ return segments.contains(si);
}
- public int indexOf(SegmentInfo si) {
- if (segmentSet.contains(si)) {
- return segments.indexOf(si);
- } else {
- return -1;
- }
+ /** WARNING: O(N) cost */
+ int indexOf(SegmentInfoPerCommit si) {
+ return segments.indexOf(si);
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Tue May 29 07:22:08 2012
@@ -46,7 +46,6 @@ import org.apache.lucene.util.ReaderUtil
*/
final class SegmentMerger {
private final Directory directory;
- private final String segment;
private final int termIndexInterval;
private final Codec codec;
@@ -54,18 +53,22 @@ final class SegmentMerger {
private final IOContext context;
private final MergeState mergeState = new MergeState();
+ private final FieldInfos.Builder fieldInfosBuilder;
- SegmentMerger(InfoStream infoStream, Directory dir, int termIndexInterval, String name, MergeState.CheckAbort checkAbort, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) {
+ // note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
+ SegmentMerger(SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, int termIndexInterval,
+ MergeState.CheckAbort checkAbort, PayloadProcessorProvider payloadProcessorProvider,
+ FieldInfos.FieldNumbers fieldNumbers, IOContext context) {
+ mergeState.segmentInfo = segmentInfo;
mergeState.infoStream = infoStream;
mergeState.readers = new ArrayList<MergeState.IndexReaderAndLiveDocs>();
- mergeState.fieldInfos = fieldInfos;
mergeState.checkAbort = checkAbort;
mergeState.payloadProcessorProvider = payloadProcessorProvider;
directory = dir;
- segment = name;
this.termIndexInterval = termIndexInterval;
- this.codec = codec;
+ this.codec = segmentInfo.getCodec();
this.context = context;
+ this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers);
}
/**
@@ -104,14 +107,14 @@ final class SegmentMerger {
// IndexWriter.close(false) takes to actually stop the
// threads.
- mergeState.mergedDocCount = setDocMaps();
-
- mergeFieldInfos();
+ mergeState.segmentInfo.setDocCount(setDocMaps());
+ mergeDocValuesAndNormsFieldInfos();
setMatchingSegmentReaders();
int numMerged = mergeFields();
- assert numMerged == mergeState.mergedDocCount;
+ assert numMerged == mergeState.segmentInfo.getDocCount();
- final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, segment, mergeState.fieldInfos, mergeState.mergedDocCount, termIndexInterval, codec, null, context);
+ final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, mergeState.segmentInfo,
+ mergeState.fieldInfos, termIndexInterval, null, context);
mergeTerms(segmentWriteState);
mergePerDoc(segmentWriteState);
@@ -121,8 +124,12 @@ final class SegmentMerger {
if (mergeState.fieldInfos.hasVectors()) {
numMerged = mergeVectors();
- assert numMerged == mergeState.mergedDocCount;
+ assert numMerged == mergeState.segmentInfo.getDocCount();
}
+
+ // write the merged infos
+ FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
+ fieldInfosWriter.write(directory, mergeState.segmentInfo.name, mergeState.fieldInfos, context);
return mergeState;
}
@@ -150,7 +157,8 @@ final class SegmentMerger {
boolean same = true;
FieldInfos segmentFieldInfos = segmentReader.getFieldInfos();
for (FieldInfo fi : segmentFieldInfos) {
- if (!mergeState.fieldInfos.fieldName(fi.number).equals(fi.name)) {
+ FieldInfo other = mergeState.fieldInfos.fieldInfo(fi.number);
+ if (other == null || !other.name.equals(fi.name)) {
same = false;
break;
}
@@ -186,17 +194,11 @@ final class SegmentMerger {
}
}
- private void mergeFieldInfos() throws IOException {
- mergeDocValuesAndNormsFieldInfos();
- // write the merged infos
- FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat()
- .getFieldInfosWriter();
- fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
- }
-
+ // NOTE: this is actually merging all the fieldinfos
public void mergeDocValuesAndNormsFieldInfos() throws IOException {
// mapping from all docvalues fields found to their promoted types
- // this is because FieldInfos does not store the valueSize
+ // this is because FieldInfos does not store the
+ // valueSize
Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
Map<FieldInfo,TypePromoter> normValuesTypes = new HashMap<FieldInfo,TypePromoter>();
@@ -204,7 +206,7 @@ final class SegmentMerger {
final AtomicReader reader = readerAndLiveDocs.reader;
FieldInfos readerFieldInfos = reader.getFieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- FieldInfo merged = mergeState.fieldInfos.add(fi);
+ FieldInfo merged = fieldInfosBuilder.add(fi);
// update the type promotion mapping for this reader
if (fi.hasDocValues()) {
TypePromoter previous = docValuesTypes.get(merged);
@@ -218,6 +220,7 @@ final class SegmentMerger {
}
updatePromoted(normValuesTypes, true);
updatePromoted(docValuesTypes, false);
+ mergeState.fieldInfos = fieldInfosBuilder.finish();
}
protected void updatePromoted(Map<FieldInfo,TypePromoter> infoAndPromoter, boolean norms) {
@@ -227,21 +230,21 @@ final class SegmentMerger {
TypePromoter promoter = e.getValue();
if (promoter == null) {
if (norms) {
- fi.setNormValueType(null, true);
+ fi.setNormValueType(null);
} else {
- fi.setDocValuesType(null, true);
+ fi.setDocValuesType(null);
}
} else {
assert promoter != TypePromoter.getIdentityPromoter();
if (norms) {
- if (fi.getNormType() != promoter.type()) {
+ if (fi.getNormType() != promoter.type() && !fi.omitsNorms()) {
// reset the type if we got promoted
- fi.setNormValueType(promoter.type(), true);
+ fi.setNormValueType(promoter.type());
}
} else {
if (fi.getDocValuesType() != promoter.type()) {
// reset the type if we got promoted
- fi.setDocValuesType(promoter.type(), true);
+ fi.setDocValuesType(promoter.type());
}
}
}
@@ -256,7 +259,7 @@ final class SegmentMerger {
* @throws IOException if there is a low-level IO error
*/
private int mergeFields() throws CorruptIndexException, IOException {
- final StoredFieldsWriter fieldsWriter = codec.storedFieldsFormat().fieldsWriter(directory, segment, context);
+ final StoredFieldsWriter fieldsWriter = codec.storedFieldsFormat().fieldsWriter(directory, mergeState.segmentInfo, context);
try {
return fieldsWriter.merge(mergeState);
@@ -270,7 +273,7 @@ final class SegmentMerger {
* @throws IOException
*/
private final int mergeVectors() throws IOException {
- final TermVectorsWriter termVectorsWriter = codec.termVectorsFormat().vectorsWriter(directory, segment, context);
+ final TermVectorsWriter termVectorsWriter = codec.termVectorsFormat().vectorsWriter(directory, mergeState.segmentInfo, context);
try {
return termVectorsWriter.merge(mergeState);
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Tue May 29 07:22:08 2012
@@ -36,7 +36,7 @@ import org.apache.lucene.util.Bits;
*/
public final class SegmentReader extends AtomicReader {
- private final SegmentInfo si;
+ private final SegmentInfoPerCommit si;
private final Bits liveDocs;
// Normally set to si.docCount - si.delDocCount, unless we
@@ -50,19 +50,19 @@ public final class SegmentReader extends
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public SegmentReader(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws IOException {
+ public SegmentReader(SegmentInfoPerCommit si, int termInfosIndexDivisor, IOContext context) throws IOException {
this.si = si;
- core = new SegmentCoreReaders(this, si.dir, si, context, termInfosIndexDivisor);
+ core = new SegmentCoreReaders(this, si.info.dir, si, context, termInfosIndexDivisor);
boolean success = false;
try {
if (si.hasDeletions()) {
// NOTE: the bitvector is stored using the regular directory, not cfs
- liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(directory(), si, new IOContext(IOContext.READ, true));
+ liveDocs = si.info.getCodec().liveDocsFormat().readLiveDocs(directory(), si, new IOContext(IOContext.READ, true));
} else {
assert si.getDelCount() == 0;
liveDocs = null;
}
- numDocs = si.docCount - si.getDelCount();
+ numDocs = si.info.getDocCount() - si.getDelCount();
success = true;
} finally {
// With lock-less commits, it's entirely possible (and
@@ -79,15 +79,17 @@ public final class SegmentReader extends
// Create new SegmentReader sharing core from a previous
// SegmentReader and loading new live docs from a new
// deletes file. Used by openIfChanged.
- SegmentReader(SegmentInfo si, SegmentCoreReaders core, IOContext context) throws IOException {
- this(si, core, si.getCodec().liveDocsFormat().readLiveDocs(si.dir, si, context), si.docCount - si.getDelCount());
+ SegmentReader(SegmentInfoPerCommit si, SegmentCoreReaders core, IOContext context) throws IOException {
+ this(si, core,
+ si.info.getCodec().liveDocsFormat().readLiveDocs(si.info.dir, si, context),
+ si.info.getDocCount() - si.getDelCount());
}
// Create new SegmentReader sharing core from a previous
// SegmentReader and using the provided in-memory
// liveDocs. Used by IndexWriter to provide a new NRT
// reader:
- SegmentReader(SegmentInfo si, SegmentCoreReaders core, Bits liveDocs, int numDocs) throws IOException {
+ SegmentReader(SegmentInfoPerCommit si, SegmentCoreReaders core, Bits liveDocs, int numDocs) throws IOException {
this.si = si;
this.core = core;
core.incRef();
@@ -151,7 +153,7 @@ public final class SegmentReader extends
@Override
public int maxDoc() {
// Don't call ensureOpen() here (it could affect performance)
- return si.docCount;
+ return si.info.getDocCount();
}
/** @lucene.internal */
@@ -179,20 +181,20 @@ public final class SegmentReader extends
public String toString() {
// SegmentInfo.toString takes dir and number of
// *pending* deletions; so we reverse compute that here:
- return si.toString(si.dir, si.docCount - numDocs - si.getDelCount());
+ return si.toString(si.info.dir, si.info.getDocCount() - numDocs - si.getDelCount());
}
/**
* Return the name of the segment this reader is reading.
*/
public String getSegmentName() {
- return si.name;
+ return si.info.name;
}
/**
- * Return the SegmentInfo of the segment this reader is reading.
+ * Return the SegmentInfoPerCommit of the segment this reader is reading.
*/
- SegmentInfo getSegmentInfo() {
+ SegmentInfoPerCommit getSegmentInfo() {
return si;
}
@@ -201,7 +203,7 @@ public final class SegmentReader extends
// Don't ensureOpen here -- in certain cases, when a
// cloned/reopened reader needs to commit, it may call
// this method on the closed original reader
- return si.dir;
+ return si.info.dir;
}
// This is necessary so that cloned SegmentReaders (which
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java Tue May 29 07:22:08 2012
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.codecs.Codec;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.InfoStream;
@@ -30,9 +29,8 @@ import org.apache.lucene.util.MutableBit
public class SegmentWriteState {
public final InfoStream infoStream;
public final Directory directory;
- public final String segmentName;
+ public final SegmentInfo segmentInfo;
public final FieldInfos fieldInfos;
- public final int numDocs;
public int delCountOnFlush;
// Deletes to apply while we are flushing the segment. A
@@ -45,7 +43,6 @@ public class SegmentWriteState {
// Lazily created:
public MutableBits liveDocs;
- public final Codec codec;
public final String segmentSuffix;
/** Expert: The fraction of terms in the "dictionary" which should be stored
@@ -57,16 +54,14 @@ public class SegmentWriteState {
public final IOContext context;
- public SegmentWriteState(InfoStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
- int numDocs, int termIndexInterval, Codec codec, BufferedDeletes segDeletes, IOContext context) {
+ public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos,
+ int termIndexInterval, BufferedDeletes segDeletes, IOContext context) {
this.infoStream = infoStream;
this.segDeletes = segDeletes;
this.directory = directory;
- this.segmentName = segmentName;
+ this.segmentInfo = segmentInfo;
this.fieldInfos = fieldInfos;
- this.numDocs = numDocs;
this.termIndexInterval = termIndexInterval;
- this.codec = codec;
segmentSuffix = "";
this.context = context;
}
@@ -77,12 +72,10 @@ public class SegmentWriteState {
public SegmentWriteState(SegmentWriteState state, String segmentSuffix) {
infoStream = state.infoStream;
directory = state.directory;
- segmentName = state.segmentName;
+ segmentInfo = state.segmentInfo;
fieldInfos = state.fieldInfos;
- numDocs = state.numDocs;
termIndexInterval = state.termIndexInterval;
context = state.context;
- codec = state.codec;
this.segmentSuffix = segmentSuffix;
segDeletes = state.segDeletes;
delCountOnFlush = state.delCountOnFlush;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Tue May 29 07:22:08 2012
@@ -89,8 +89,8 @@ final class StandardDirectoryReader exte
IOException prior = null;
boolean success = false;
try {
- final SegmentInfo info = infos.info(i);
- assert info.dir == dir;
+ final SegmentInfoPerCommit info = infos.info(i);
+ assert info.info.dir == dir;
final ReadersAndLiveDocs rld = writer.readerPool.get(info, true);
try {
final SegmentReader reader = rld.getReadOnlyClone(IOContext.READ);
@@ -140,7 +140,7 @@ final class StandardDirectoryReader exte
for (int i = infos.size() - 1; i>=0; i--) {
// find SegmentReader for this segment
- Integer oldReaderIndex = segmentReaders.get(infos.info(i).name);
+ Integer oldReaderIndex = segmentReaders.get(infos.info(i).info.name);
if (oldReaderIndex == null) {
// this is a new segment, no old SegmentReader can be reused
newReaders[i] = null;
@@ -153,7 +153,7 @@ final class StandardDirectoryReader exte
IOException prior = null;
try {
SegmentReader newReader;
- if (newReaders[i] == null || infos.info(i).getUseCompoundFile() != newReaders[i].getSegmentInfo().getUseCompoundFile()) {
+ if (newReaders[i] == null || infos.info(i).info.getUseCompoundFile() != newReaders[i].getSegmentInfo().info.getUseCompoundFile()) {
// this is a new reader; in case we hit an exception we can close it safely
newReader = new SegmentReader(infos.info(i), termInfosIndexDivisor, IOContext.READ);
@@ -169,7 +169,7 @@ final class StandardDirectoryReader exte
} else {
readerShared[i] = false;
// Steal the ref returned by SegmentReader ctor:
- assert infos.info(i).dir == newReaders[i].getSegmentInfo().dir;
+ assert infos.info(i).info.dir == newReaders[i].getSegmentInfo().info.dir;
assert infos.info(i).hasDeletions();
newReaders[i] = new SegmentReader(infos.info(i), newReaders[i].core, IOContext.READ);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java Tue May 29 07:22:08 2012
@@ -58,18 +58,19 @@ final class StoredFieldsConsumer {
}
public void flush(SegmentWriteState state) throws IOException {
+ int numDocs = state.segmentInfo.getDocCount();
- if (state.numDocs > 0) {
+ if (numDocs > 0) {
// It's possible that all documents seen in this segment
// hit non-aborting exceptions, in which case we will
// not have yet init'd the FieldsWriter:
initFieldsWriter(state.context);
- fill(state.numDocs);
+ fill(numDocs);
}
if (fieldsWriter != null) {
try {
- fieldsWriter.finish(state.numDocs);
+ fieldsWriter.finish(state.fieldInfos, numDocs);
} finally {
fieldsWriter.close();
fieldsWriter = null;
@@ -80,7 +81,7 @@ final class StoredFieldsConsumer {
private synchronized void initFieldsWriter(IOContext context) throws IOException {
if (fieldsWriter == null) {
- fieldsWriter = codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context);
+ fieldsWriter = codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
lastDocID = 0;
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java Tue May 29 07:22:08 2012
@@ -49,13 +49,14 @@ final class TermVectorsConsumer extends
}
@Override
- void flush(Map<FieldInfo, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ void flush(Map<String, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
if (writer != null) {
+ int numDocs = state.segmentInfo.getDocCount();
// At least one doc in this run had term vectors enabled
try {
- fill(state.numDocs);
- assert state.segmentName != null;
- writer.finish(state.numDocs);
+ fill(numDocs);
+ assert state.segmentInfo != null;
+ writer.finish(state.fieldInfos, numDocs);
} finally {
IOUtils.close(writer);
writer = null;
@@ -84,7 +85,7 @@ final class TermVectorsConsumer extends
private final void initTermVectorsWriter() throws IOException {
if (writer == null) {
IOContext context = new IOContext(new FlushInfo(docWriter.getNumDocsInRAM(), docWriter.bytesUsed()));
- writer = docWriter.codec.termVectorsFormat().vectorsWriter(docWriter.directory, docWriter.getSegment(), context);
+ writer = docWriter.codec.termVectorsFormat().vectorsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
lastDocID = 0;
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Tue May 29 07:22:08 2012
@@ -91,8 +91,9 @@ final class TermVectorsConsumerPerField
* RAMOutputStream, which is then quickly flushed to
* the real term vectors files in the Directory. */ @Override
void finish() throws IOException {
- if (!doVectors || termsHashPerField.bytesHash.size() == 0)
+ if (!doVectors || termsHashPerField.bytesHash.size() == 0) {
return;
+ }
termsWriter.addFieldToFlush(this);
}
@@ -148,7 +149,7 @@ final class TermVectorsConsumerPerField
termsHashPerField.reset();
- // commit the termVectors once successful success - FI will otherwise reset them
+ // commit the termVectors once successful - FI will otherwise reset them
fieldInfo.setStoreTermVectors();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Tue May 29 07:22:08 2012
@@ -96,17 +96,17 @@ final class TermsHash extends InvertedDo
}
@Override
- void flush(Map<FieldInfo,InvertedDocConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
- Map<FieldInfo,TermsHashConsumerPerField> childFields = new HashMap<FieldInfo,TermsHashConsumerPerField>();
- Map<FieldInfo,InvertedDocConsumerPerField> nextChildFields;
+ void flush(Map<String,InvertedDocConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ Map<String,TermsHashConsumerPerField> childFields = new HashMap<String,TermsHashConsumerPerField>();
+ Map<String,InvertedDocConsumerPerField> nextChildFields;
if (nextTermsHash != null) {
- nextChildFields = new HashMap<FieldInfo,InvertedDocConsumerPerField>();
+ nextChildFields = new HashMap<String,InvertedDocConsumerPerField>();
} else {
nextChildFields = null;
}
- for (final Map.Entry<FieldInfo,InvertedDocConsumerPerField> entry : fieldsToFlush.entrySet()) {
+ for (final Map.Entry<String,InvertedDocConsumerPerField> entry : fieldsToFlush.entrySet()) {
TermsHashPerField perField = (TermsHashPerField) entry.getValue();
childFields.put(entry.getKey(), perField.consumer);
if (nextTermsHash != null) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java?rev=1343568&r1=1343567&r2=1343568&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java Tue May 29 07:22:08 2012
@@ -21,7 +21,7 @@ import java.io.IOException;
import java.util.Map;
abstract class TermsHashConsumer {
- abstract void flush(Map<FieldInfo, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException;
+ abstract void flush(Map<String, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException;
abstract void abort();
abstract void startDocument() throws IOException;
abstract void finishDocument(TermsHash termsHash) throws IOException;