You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/11/01 07:03:54 UTC
svn commit: r1537832 [3/5] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/cod...
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Fri Nov 1 06:03:52 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
@@ -29,17 +28,19 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xSegmentInfoReader;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.ChecksumIndexOutput;
-import org.apache.lucene.store.DataOutput; // javadocs
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -47,7 +48,6 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.ThreadInterruptedException;
/**
* A collection of segmentInfo objects with methods for operating on
@@ -75,7 +75,7 @@ import org.apache.lucene.util.ThreadInte
* <ul>
* <li><tt>segments.gen</tt>: GenHeader, Generation, Generation
* <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount,
- * <SegName, SegCodec, DelGen, DeletionCount><sup>SegCount</sup>,
+ * <SegName, SegCodec, DelGen, DeletionCount, FieldInfosGen, UpdatesFiles><sup>SegCount</sup>,
* CommitUserData, Checksum
* </ul>
* </p>
@@ -84,9 +84,10 @@ import org.apache.lucene.util.ThreadInte
* <ul>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>GenHeader, NameCounter, SegCount, DeletionCount --> {@link DataOutput#writeInt Int32}</li>
- * <li>Generation, Version, DelGen, Checksum --> {@link DataOutput#writeLong Int64}</li>
+ * <li>Generation, Version, DelGen, Checksum, FieldInfosGen --> {@link DataOutput#writeLong Int64}</li>
* <li>SegName, SegCodec --> {@link DataOutput#writeString String}</li>
* <li>CommitUserData --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
+ * <li>UpdatesFiles --> {@link DataOutput#writeStringSet(Set) Set<String>}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -109,6 +110,10 @@ import org.apache.lucene.util.ThreadInte
* <li>CommitUserData stores an optional user-supplied opaque
* Map<String,String> that was passed to
* {@link IndexWriter#setCommitData(java.util.Map)}.</li>
+ * <li>FieldInfosGen is the generation count of the fieldInfos file. If this is -1,
+ * there are no updates to the fieldInfos in that segment. Anything above zero
+ * means there are updates to fieldInfos stored by {@link FieldInfosFormat}.</li>
+ * <li>UpdatesFiles stores the list of files that were updated in that segment.</li>
* </ul>
* </p>
*
@@ -116,11 +121,12 @@ import org.apache.lucene.util.ThreadInte
*/
public final class SegmentInfos implements Cloneable, Iterable<SegmentInfoPerCommit> {
- /**
- * The file format version for the segments_N codec header
- */
+ /** The file format version for the segments_N codec header, up to 4.5. */
public static final int VERSION_40 = 0;
+ /** The file format version for the segments_N codec header, since 4.6+. */
+ public static final int VERSION_46 = 1;
+
/** Used for the segments.gen file only!
* Whenever you add a new format, make it 1 smaller (negative version logic)! */
public static final int FORMAT_SEGMENTS_GEN_CURRENT = -2;
@@ -320,7 +326,7 @@ public final class SegmentInfos implemen
final int format = input.readInt();
if (format == CodecUtil.CODEC_MAGIC) {
// 4.0+
- CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_40);
+ int actualFormat = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_46);
version = input.readLong();
counter = input.readInt();
int numSegments = input.readInt();
@@ -338,7 +344,25 @@ public final class SegmentInfos implemen
if (delCount < 0 || delCount > info.getDocCount()) {
throw new CorruptIndexException("invalid deletion count: " + delCount + " (resource: " + input + ")");
}
- add(new SegmentInfoPerCommit(info, delCount, delGen));
+ long fieldInfosGen = -1;
+ if (actualFormat >= VERSION_46) {
+ fieldInfosGen = input.readLong();
+ }
+ SegmentInfoPerCommit siPerCommit = new SegmentInfoPerCommit(info, delCount, delGen, fieldInfosGen);
+ if (actualFormat >= VERSION_46) {
+ int numGensUpdatesFiles = input.readInt();
+ final Map<Long,Set<String>> genUpdatesFiles;
+ if (numGensUpdatesFiles == 0) {
+ genUpdatesFiles = Collections.emptyMap();
+ } else {
+ genUpdatesFiles = new HashMap<Long,Set<String>>(numGensUpdatesFiles);
+ for (int i = 0; i < numGensUpdatesFiles; i++) {
+ genUpdatesFiles.put(input.readLong(), input.readStringSet());
+ }
+ }
+ siPerCommit.setGenUpdatesFiles(genUpdatesFiles);
+ }
+ add(siPerCommit);
}
userData = input.readStringStringMap();
} else {
@@ -408,7 +432,7 @@ public final class SegmentInfos implemen
try {
segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentsFileName, IOContext.DEFAULT));
- CodecUtil.writeHeader(segnOutput, "segments", VERSION_40);
+ CodecUtil.writeHeader(segnOutput, "segments", VERSION_46);
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
@@ -418,6 +442,13 @@ public final class SegmentInfos implemen
segnOutput.writeString(si.getCodec().getName());
segnOutput.writeLong(siPerCommit.getDelGen());
segnOutput.writeInt(siPerCommit.getDelCount());
+ segnOutput.writeLong(siPerCommit.getFieldInfosGen());
+ final Map<Long,Set<String>> genUpdatesFiles = siPerCommit.getUpdatesFiles();
+ segnOutput.writeInt(genUpdatesFiles.size());
+ for (Entry<Long,Set<String>> e : genUpdatesFiles.entrySet()) {
+ segnOutput.writeLong(e.getKey());
+ segnOutput.writeStringSet(e.getValue());
+ }
assert si.dir == directory;
assert siPerCommit.getDelCount() <= si.getDocCount();
@@ -931,6 +962,7 @@ public final class SegmentInfos implemen
files.addAll(info.files());
}
}
+
return files;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Fri Nov 1 06:03:52 2013
@@ -145,7 +145,7 @@ final class SegmentMerger {
// write the merged infos
FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- fieldInfosWriter.write(directory, mergeState.segmentInfo.name, mergeState.fieldInfos, context);
+ fieldInfosWriter.write(directory, mergeState.segmentInfo.name, "", mergeState.fieldInfos, context);
return mergeState;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Fri Nov 1 06:03:52 2013
@@ -18,13 +18,24 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-
-import org.apache.lucene.store.Directory;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.search.FieldCache; // javadocs
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.CloseableThreadLocal;
/**
* IndexReader implementation over a single segment.
@@ -44,7 +55,28 @@ public final class SegmentReader extends
private final int numDocs;
final SegmentCoreReaders core;
+ final SegmentDocValues segDocValues;
+
+ final CloseableThreadLocal<Map<String,Object>> docValuesLocal = new CloseableThreadLocal<Map<String,Object>>() {
+ @Override
+ protected Map<String,Object> initialValue() {
+ return new HashMap<String,Object>();
+ }
+ };
+
+ final CloseableThreadLocal<Map<String,Bits>> docsWithFieldLocal = new CloseableThreadLocal<Map<String,Bits>>() {
+ @Override
+ protected Map<String,Bits> initialValue() {
+ return new HashMap<String,Bits>();
+ }
+ };
+ final Map<String,DocValuesProducer> dvProducers = new HashMap<String,DocValuesProducer>();
+
+ final FieldInfos fieldInfos;
+
+ private final List<Long> dvGens = new ArrayList<Long>();
+
/**
* Constructs a new SegmentReader with a new core.
* @throws CorruptIndexException if the index is corrupt
@@ -53,17 +85,32 @@ public final class SegmentReader extends
// TODO: why is this public?
public SegmentReader(SegmentInfoPerCommit si, int termInfosIndexDivisor, IOContext context) throws IOException {
this.si = si;
+ // TODO if the segment uses CFS, we may open the CFS file twice: once for
+ // reading the FieldInfos (if they are not gen'd) and second time by
+ // SegmentCoreReaders. We can open the CFS here and pass to SCR, but then it
+ // results in less readable code (resource not closed where it was opened).
+ // Best if we could somehow read FieldInfos in SCR but not keep it there, but
+ // constructors don't allow returning two things...
+ fieldInfos = readFieldInfos(si);
core = new SegmentCoreReaders(this, si.info.dir, si, context, termInfosIndexDivisor);
+ segDocValues = new SegmentDocValues();
+
boolean success = false;
+ final Codec codec = si.info.getCodec();
try {
if (si.hasDeletions()) {
// NOTE: the bitvector is stored using the regular directory, not cfs
- liveDocs = si.info.getCodec().liveDocsFormat().readLiveDocs(directory(), si, IOContext.READONCE);
+ liveDocs = codec.liveDocsFormat().readLiveDocs(directory(), si, IOContext.READONCE);
} else {
assert si.getDelCount() == 0;
liveDocs = null;
}
numDocs = si.info.getDocCount() - si.getDelCount();
+
+ if (fieldInfos.hasDocValues()) {
+ initDocValuesProducers(codec);
+ }
+
success = true;
} finally {
// With lock-less commits, it's entirely possible (and
@@ -72,7 +119,7 @@ public final class SegmentReader extends
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success) {
- core.decRef();
+ doClose();
}
}
}
@@ -80,8 +127,8 @@ public final class SegmentReader extends
/** Create new SegmentReader sharing core from a previous
* SegmentReader and loading new live docs from a new
* deletes file. Used by openIfChanged. */
- SegmentReader(SegmentInfoPerCommit si, SegmentCoreReaders core) throws IOException {
- this(si, core,
+ SegmentReader(SegmentInfoPerCommit si, SegmentReader sr) throws IOException {
+ this(si, sr,
si.info.getCodec().liveDocsFormat().readLiveDocs(si.info.dir, si, IOContext.READONCE),
si.info.getDocCount() - si.getDelCount());
}
@@ -90,15 +137,104 @@ public final class SegmentReader extends
* SegmentReader and using the provided in-memory
* liveDocs. Used by IndexWriter to provide a new NRT
* reader */
- SegmentReader(SegmentInfoPerCommit si, SegmentCoreReaders core, Bits liveDocs, int numDocs) {
+ SegmentReader(SegmentInfoPerCommit si, SegmentReader sr, Bits liveDocs, int numDocs) throws IOException {
this.si = si;
- this.core = core;
- core.incRef();
-
- assert liveDocs != null;
this.liveDocs = liveDocs;
-
this.numDocs = numDocs;
+ this.core = sr.core;
+ core.incRef();
+ this.segDocValues = sr.segDocValues;
+
+// System.out.println("[" + Thread.currentThread().getName() + "] SR.init: sharing reader: " + sr + " for gens=" + sr.genDVProducers.keySet());
+
+ // increment refCount of DocValuesProducers that are used by this reader
+ boolean success = false;
+ try {
+ final Codec codec = si.info.getCodec();
+ if (si.getFieldInfosGen() == -1) {
+ fieldInfos = sr.fieldInfos;
+ } else {
+ fieldInfos = readFieldInfos(si);
+ }
+
+ if (fieldInfos.hasDocValues()) {
+ initDocValuesProducers(codec);
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ doClose();
+ }
+ }
+ }
+
+ // initialize the per-field DocValuesProducer
+ private void initDocValuesProducers(Codec codec) throws IOException {
+ final Directory dir = core.cfsReader != null ? core.cfsReader : si.info.dir;
+ final DocValuesFormat dvFormat = codec.docValuesFormat();
+ final Map<Long,List<FieldInfo>> genInfos = getGenInfos();
+
+// System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gens=" + genInfos.keySet());
+
+ for (Entry<Long,List<FieldInfo>> e : genInfos.entrySet()) {
+ Long gen = e.getKey();
+ List<FieldInfo> infos = e.getValue();
+ DocValuesProducer dvp = segDocValues.getDocValuesProducer(gen, si, IOContext.READ, dir, dvFormat, infos, getTermInfosIndexDivisor());
+ for (FieldInfo fi : infos) {
+ dvProducers.put(fi.name, dvp);
+ }
+ }
+
+ dvGens.addAll(genInfos.keySet());
+ }
+
+ /**
+ * Reads the most recent {@link FieldInfos} of the given segment info.
+ *
+ * @lucene.internal
+ */
+ static FieldInfos readFieldInfos(SegmentInfoPerCommit info) throws IOException {
+ final Directory dir;
+ final boolean closeDir;
+ if (info.getFieldInfosGen() == -1 && info.info.getUseCompoundFile()) {
+ // no fieldInfos gen and segment uses a compound file
+ dir = new CompoundFileDirectory(info.info.dir,
+ IndexFileNames.segmentFileName(info.info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION),
+ IOContext.READONCE,
+ false);
+ closeDir = true;
+ } else {
+ // gen'd FIS are read outside CFS, or the segment doesn't use a compound file
+ dir = info.info.dir;
+ closeDir = false;
+ }
+
+ try {
+ final String segmentSuffix = info.getFieldInfosGen() == -1 ? "" : Long.toString(info.getFieldInfosGen(), Character.MAX_RADIX);
+ return info.info.getCodec().fieldInfosFormat().getFieldInfosReader().read(dir, info.info.name, segmentSuffix, IOContext.READONCE);
+ } finally {
+ if (closeDir) {
+ dir.close();
+ }
+ }
+ }
+
+ // returns a gen->List<FieldInfo> mapping. Fields without DV updates have gen=-1
+ private Map<Long,List<FieldInfo>> getGenInfos() {
+ final Map<Long,List<FieldInfo>> genInfos = new HashMap<Long,List<FieldInfo>>();
+ for (FieldInfo fi : fieldInfos) {
+ if (fi.getDocValuesType() == null) {
+ continue;
+ }
+ long gen = fi.getDocValuesGen();
+ List<FieldInfo> infos = genInfos.get(gen);
+ if (infos == null) {
+ infos = new ArrayList<FieldInfo>();
+ genInfos.put(gen, infos);
+ }
+ infos.add(fi);
+ }
+ return genInfos;
}
@Override
@@ -110,13 +246,20 @@ public final class SegmentReader extends
@Override
protected void doClose() throws IOException {
//System.out.println("SR.close seg=" + si);
- core.decRef();
+ try {
+ core.decRef();
+ } finally {
+ dvProducers.clear();
+ docValuesLocal.close();
+ docsWithFieldLocal.close();
+ segDocValues.decRef(dvGens);
+ }
}
@Override
public FieldInfos getFieldInfos() {
ensureOpen();
- return core.fieldInfos;
+ return fieldInfos;
}
/** Expert: retrieve thread-private {@link
@@ -226,40 +369,151 @@ public final class SegmentReader extends
return core.termsIndexDivisor;
}
+ // returns the FieldInfo that corresponds to the given field and type, or
+ // null if the field does not exist, or not indexed as the requested
+ // DovDocValuesType.
+ private FieldInfo getDVField(String field, DocValuesType type) {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+ if (fi.getDocValuesType() != type) {
+ // Field DocValues are different than requested type
+ return null;
+ }
+
+ return fi;
+ }
+
@Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
ensureOpen();
- return core.getNumericDocValues(field);
+ FieldInfo fi = getDVField(field, DocValuesType.NUMERIC);
+ if (fi == null) {
+ return null;
+ }
+
+ DocValuesProducer dvProducer = dvProducers.get(field);
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ NumericDocValues dvs = (NumericDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getNumeric(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
}
@Override
public Bits getDocsWithField(String field) throws IOException {
ensureOpen();
- return core.getDocsWithField(field);
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+
+ DocValuesProducer dvProducer = dvProducers.get(field);
+ assert dvProducer != null;
+
+ Map<String,Bits> dvFields = docsWithFieldLocal.get();
+
+ Bits dvs = dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getDocsWithField(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
}
@Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
ensureOpen();
- return core.getBinaryDocValues(field);
+ FieldInfo fi = getDVField(field, DocValuesType.BINARY);
+ if (fi == null) {
+ return null;
+ }
+
+ DocValuesProducer dvProducer = dvProducers.get(field);
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ BinaryDocValues dvs = (BinaryDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getBinary(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
}
@Override
public SortedDocValues getSortedDocValues(String field) throws IOException {
ensureOpen();
- return core.getSortedDocValues(field);
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED);
+ if (fi == null) {
+ return null;
+ }
+
+ DocValuesProducer dvProducer = dvProducers.get(field);
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ SortedDocValues dvs = (SortedDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getSorted(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
}
@Override
public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
ensureOpen();
- return core.getSortedSetDocValues(field);
+ FieldInfo fi = getDVField(field, DocValuesType.SORTED_SET);
+ if (fi == null) {
+ return null;
+ }
+
+ DocValuesProducer dvProducer = dvProducers.get(field);
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ SortedSetDocValues dvs = (SortedSetDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getSortedSet(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
}
@Override
public NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
- return core.getNormValues(field);
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null || !fi.hasNorms()) {
+ // Field does not exist or does not index norms
+ return null;
+ }
+ return core.getNormValues(fi);
}
/**
@@ -296,6 +550,13 @@ public final class SegmentReader extends
/** Returns approximate RAM Bytes used */
public long ramBytesUsed() {
ensureOpen();
- return (core!=null) ? core.ramBytesUsed() : 0;
+ long ramBytesUsed = 0;
+ if (segDocValues != null) {
+ ramBytesUsed += segDocValues.ramBytesUsed();
+ }
+ if (core != null) {
+ ramBytesUsed += core.ramBytesUsed();
+ }
+ return ramBytesUsed;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java Fri Nov 1 06:03:52 2013
@@ -81,13 +81,24 @@ public class SegmentWriteState {
/** Sole constructor. */
public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos,
int termIndexInterval, BufferedDeletes segDeletes, IOContext context) {
+ this(infoStream, directory, segmentInfo, fieldInfos, termIndexInterval, segDeletes, context, "");
+ }
+
+ /**
+ * Constructor which takes segment suffix.
+ *
+ * @see #SegmentWriteState(InfoStream, Directory, SegmentInfo, FieldInfos, int,
+ * BufferedDeletes, IOContext)
+ */
+ public SegmentWriteState(InfoStream infoStream, Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos,
+ int termIndexInterval, BufferedDeletes segDeletes, IOContext context, String segmentSuffix) {
this.infoStream = infoStream;
this.segDeletes = segDeletes;
this.directory = directory;
this.segmentInfo = segmentInfo;
this.fieldInfos = fieldInfos;
this.termIndexInterval = termIndexInterval;
- segmentSuffix = "";
+ this.segmentSuffix = segmentSuffix;
this.context = context;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Fri Nov 1 06:03:52 2013
@@ -173,18 +173,26 @@ final class StandardDirectoryReader exte
readerShared[i] = false;
newReaders[i] = newReader;
} else {
- if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()) {
+ if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()
+ && newReaders[i].getSegmentInfo().getFieldInfosGen() == infos.info(i).getFieldInfosGen()) {
// No change; this reader will be shared between
// the old and the new one, so we must incRef
// it:
readerShared[i] = true;
newReaders[i].incRef();
} else {
+ // there are changes to the reader, either liveDocs or DV updates
readerShared[i] = false;
// Steal the ref returned by SegmentReader ctor:
assert infos.info(i).info.dir == newReaders[i].getSegmentInfo().info.dir;
- assert infos.info(i).hasDeletions();
- newReaders[i] = new SegmentReader(infos.info(i), newReaders[i].core);
+ assert infos.info(i).hasDeletions() || infos.info(i).hasFieldUpdates();
+ if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()) {
+ // only DV updates
+ newReaders[i] = new SegmentReader(infos.info(i), newReaders[i], newReaders[i].getLiveDocs(), newReaders[i].numDocs());
+ } else {
+ // both DV and liveDocs have changed
+ newReaders[i] = new SegmentReader(infos.info(i), newReaders[i]);
+ }
}
}
success = true;
Copied: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RefCount.java (from r1523461, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/RefCount.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RefCount.java?p2=lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RefCount.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/RefCount.java&r1=1523461&r2=1537832&rev=1537832&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/RefCount.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RefCount.java Fri Nov 1 06:03:52 2013
@@ -1,84 +1,84 @@
-package org.apache.lucene.util;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Manages reference counting for a given object. Extensions can override
- * {@link #release()} to do custom logic when reference counting hits 0.
- */
-public class RefCount<T> {
-
- private final AtomicInteger refCount = new AtomicInteger(1);
-
- protected final T object;
-
- public RefCount(T object) {
- this.object = object;
- }
-
- /**
- * Called when reference counting hits 0. By default this method does nothing,
- * but extensions can override to e.g. release resources attached to object
- * that is managed by this class.
- */
- protected void release() throws IOException {}
-
- /**
- * Decrements the reference counting of this object. When reference counting
- * hits 0, calls {@link #release()}.
- */
- public final void decRef() throws IOException {
- final int rc = refCount.decrementAndGet();
- if (rc == 0) {
- boolean success = false;
- try {
- release();
- success = true;
- } finally {
- if (!success) {
- // Put reference back on failure
- refCount.incrementAndGet();
- }
- }
- } else if (rc < 0) {
- throw new IllegalStateException("too many decRef calls: refCount is " + rc + " after decrement");
- }
- }
-
- public final T get() {
- return object;
- }
-
- /** Returns the current reference count. */
- public final int getRefCount() {
- return refCount.get();
- }
-
- /**
- * Increments the reference count. Calls to this method must be matched with
- * calls to {@link #decRef()}.
- */
- public final void incRef() {
- refCount.incrementAndGet();
- }
-
-}
-
+package org.apache.lucene.util;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Manages reference counting for a given object. Extensions can override
+ * {@link #release()} to do custom logic when reference counting hits 0.
+ */
+public class RefCount<T> {
+
+ private final AtomicInteger refCount = new AtomicInteger(1);
+
+ protected final T object;
+
+ public RefCount(T object) {
+ this.object = object;
+ }
+
+ /**
+ * Called when reference counting hits 0. By default this method does nothing,
+ * but extensions can override to e.g. release resources attached to object
+ * that is managed by this class.
+ */
+ protected void release() throws IOException {}
+
+ /**
+ * Decrements the reference counting of this object. When reference counting
+ * hits 0, calls {@link #release()}.
+ */
+ public final void decRef() throws IOException {
+ final int rc = refCount.decrementAndGet();
+ if (rc == 0) {
+ boolean success = false;
+ try {
+ release();
+ success = true;
+ } finally {
+ if (!success) {
+ // Put reference back on failure
+ refCount.incrementAndGet();
+ }
+ }
+ } else if (rc < 0) {
+ throw new IllegalStateException("too many decRef calls: refCount is " + rc + " after decrement");
+ }
+ }
+
+ public final T get() {
+ return object;
+ }
+
+ /** Returns the current reference count. */
+ public final int getRefCount() {
+ return refCount.get();
+ }
+
+ /**
+ * Increments the reference count. Calls to this method must be matched with
+ * calls to {@link #decRef()}.
+ */
+ public final void incRef() {
+ refCount.incrementAndGet();
+ }
+
+}
+
Modified: lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec Fri Nov 1 06:03:52 2013
@@ -17,4 +17,5 @@ org.apache.lucene.codecs.lucene40.Lucene
org.apache.lucene.codecs.lucene3x.Lucene3xCodec
org.apache.lucene.codecs.lucene41.Lucene41Codec
org.apache.lucene.codecs.lucene42.Lucene42Codec
-org.apache.lucene.codecs.lucene45.Lucene45Codec
\ No newline at end of file
+org.apache.lucene.codecs.lucene45.Lucene45Codec
+org.apache.lucene.codecs.lucene46.Lucene46Codec
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java Fri Nov 1 06:03:52 2013
@@ -19,7 +19,7 @@ package org.apache.lucene;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene45.Lucene45Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -37,7 +37,7 @@ import org.apache.lucene.util.LuceneTest
public class TestExternalCodecs extends LuceneTestCase {
- private static final class CustomPerFieldCodec extends Lucene45Codec {
+ private static final class CustomPerFieldCodec extends Lucene46Codec {
private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly");
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene3x/TestTermInfosReaderIndex.java Fri Nov 1 06:03:52 2013
@@ -97,7 +97,7 @@ public class TestTermInfosReaderIndex ex
r.close();
FieldInfosReader infosReader = new PreFlexRWCodec().fieldInfosFormat().getFieldInfosReader();
- FieldInfos fieldInfos = infosReader.read(directory, segment, IOContext.READONCE);
+ FieldInfos fieldInfos = infosReader.read(directory, segment, "", IOContext.READONCE);
String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION);
long tiiFileLength = directory.fileLength(segmentFileName);
IndexInput input = directory.openInput(segmentFileName, newIOContext(random()));
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java Fri Nov 1 06:03:52 2013
@@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.lucene45.Lucene45Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -80,7 +80,7 @@ public class TestPerFieldDocValuesFormat
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
final DocValuesFormat fast = DocValuesFormat.forName("Lucene45");
final DocValuesFormat slow = DocValuesFormat.forName("SimpleText");
- iwc.setCodec(new Lucene45Codec() {
+ iwc.setCodec(new Lucene46Codec() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
if ("dv1".equals(field)) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java Fri Nov 1 06:03:52 2013
@@ -21,8 +21,8 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
@@ -34,10 +34,10 @@ import org.apache.lucene.index.Directory
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@@ -200,7 +200,7 @@ public class TestPerFieldPostingsFormat2
}
- public static class MockCodec extends Lucene45Codec {
+ public static class MockCodec extends Lucene46Codec {
final PostingsFormat lucene40 = new Lucene41PostingsFormat();
final PostingsFormat simpleText = new SimpleTextPostingsFormat();
final PostingsFormat mockSep = new MockSepPostingsFormat();
@@ -217,7 +217,7 @@ public class TestPerFieldPostingsFormat2
}
}
- public static class MockCodec2 extends Lucene45Codec {
+ public static class MockCodec2 extends Lucene46Codec {
final PostingsFormat lucene40 = new Lucene41PostingsFormat();
final PostingsFormat simpleText = new SimpleTextPostingsFormat();
@@ -268,7 +268,7 @@ public class TestPerFieldPostingsFormat2
}
public void testSameCodecDifferentInstance() throws Exception {
- Codec codec = new Lucene45Codec() {
+ Codec codec = new Lucene46Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if ("id".equals(field)) {
@@ -284,7 +284,7 @@ public class TestPerFieldPostingsFormat2
}
public void testSameCodecDifferentParams() throws Exception {
- Codec codec = new Lucene45Codec() {
+ Codec codec = new Lucene46Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if ("id".equals(field)) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Nov 1 06:03:52 2013
@@ -27,7 +27,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene45.Lucene45Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -42,7 +42,6 @@ import org.apache.lucene.store.BaseDirec
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -1059,7 +1058,7 @@ public class TestAddIndexes extends Luce
aux2.close();
}
- private static final class CustomPerFieldCodec extends Lucene45Codec {
+ private static final class CustomPerFieldCodec extends Lucene46Codec {
private final PostingsFormat simpleTextFormat = PostingsFormat.forName("SimpleText");
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
private final PostingsFormat mockSepFormat = PostingsFormat.forName("MockSep");
@@ -1110,7 +1109,7 @@ public class TestAddIndexes extends Luce
private static final class UnRegisteredCodec extends FilterCodec {
public UnRegisteredCodec() {
- super("NotRegistered", new Lucene45Codec());
+ super("NotRegistered", new Lucene46Codec());
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Fri Nov 1 06:03:52 2013
@@ -21,12 +21,9 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.lucene45.Lucene45Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -41,7 +38,7 @@ public class TestAllFilesHaveCodecHeader
public void test() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- conf.setCodec(new Lucene45Codec());
+ conf.setCodec(new Lucene46Codec());
// riw should sometimes create docvalues fields, etc
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
Document doc = new Document();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Fri Nov 1 06:03:52 2013
@@ -37,8 +37,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
@@ -54,14 +54,14 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
@@ -76,7 +76,7 @@ import org.junit.BeforeClass;
// we won't even be running the actual code, only the impostor
// @SuppressCodecs("Lucene4x")
// Sep codec cannot yet handle the offsets in our 4.x index!
-@SuppressCodecs({"Lucene3x", "MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene40", "Lucene41", "Appending", "Lucene42"})
+@SuppressCodecs({"Lucene3x", "MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene40", "Lucene41", "Appending", "Lucene42", "Lucene45"})
public class TestBackwardsCompatibility extends LuceneTestCase {
// Uncomment these cases & run them on an older Lucene version,
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Fri Nov 1 06:03:52 2013
@@ -299,7 +299,7 @@ public class TestCodecs extends LuceneTe
final Directory dir = newDirectory();
this.write(fieldInfos, dir, fields, true);
Codec codec = Codec.getDefault();
- final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null);
+ final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null);
final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -356,8 +356,7 @@ public class TestCodecs extends LuceneTe
this.write(fieldInfos, dir, fields, false);
Codec codec = Codec.getDefault();
- final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000,
- false, codec, null, null);
+ final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null);
if (VERBOSE) {
System.out.println("TEST: now read postings");
@@ -666,7 +665,7 @@ public class TestCodecs extends LuceneTe
final int termIndexInterval = _TestUtil.nextInt(random(), 13, 27);
final Codec codec = Codec.getDefault();
- final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null, null);
+ final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, false, codec, null);
final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, termIndexInterval, null, newIOContext(random()));
final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Fri Nov 1 06:03:52 2013
@@ -29,7 +29,6 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FailOnNonBulkMergesInfoStream;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import org.junit.Test;
public class TestConsistentFieldNumbers extends LuceneTestCase {
@@ -67,8 +66,8 @@ public class TestConsistentFieldNumbers
sis.read(dir);
assertEquals(2, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
@@ -85,7 +84,7 @@ public class TestConsistentFieldNumbers
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis3 = _TestUtil.getFieldInfos(sis.info(0).info);
+ FieldInfos fis3 = SegmentReader.readFieldInfos(sis.info(0));
assertEquals("f1", fis3.fieldInfo(0).name);
assertEquals("f2", fis3.fieldInfo(1).name);
@@ -130,8 +129,8 @@ public class TestConsistentFieldNumbers
sis.read(dir1);
assertEquals(2, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
@@ -161,7 +160,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
}
@@ -180,8 +179,8 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(2, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f1", fis2.fieldInfo(0).name);
@@ -203,9 +202,9 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(3, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
- FieldInfos fis2 = _TestUtil.getFieldInfos(sis.info(1).info);
- FieldInfos fis3 = _TestUtil.getFieldInfos(sis.info(2).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
+ FieldInfos fis2 = SegmentReader.readFieldInfos(sis.info(1));
+ FieldInfos fis3 = SegmentReader.readFieldInfos(sis.info(2));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f1", fis2.fieldInfo(0).name);
@@ -237,7 +236,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
assertEquals(1, sis.size());
- FieldInfos fis1 = _TestUtil.getFieldInfos(sis.info(0).info);
+ FieldInfos fis1 = SegmentReader.readFieldInfos(sis.info(0));
assertEquals("f1", fis1.fieldInfo(0).name);
assertEquals("f2", fis1.fieldInfo(1).name);
assertEquals("f3", fis1.fieldInfo(2).name);
@@ -275,7 +274,7 @@ public class TestConsistentFieldNumbers
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
for (SegmentInfoPerCommit si : sis) {
- FieldInfos fis = _TestUtil.getFieldInfos(si.info);
+ FieldInfos fis = SegmentReader.readFieldInfos(si);
for (FieldInfo fi : fis) {
Field expected = getField(Integer.parseInt(fi.name));
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Fri Nov 1 06:03:52 2013
@@ -217,7 +217,7 @@ public class TestDoc extends LuceneTestC
final Codec codec = Codec.getDefault();
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
- final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, false, codec, null, null);
+ final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, false, codec, null);
SegmentMerger merger = new SegmentMerger(Arrays.<AtomicReader>asList(r1, r2),
si, InfoStream.getDefault(), trackingDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
@@ -228,7 +228,7 @@ public class TestDoc extends LuceneTestC
r2.close();
final SegmentInfo info = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged,
si1.info.getDocCount() + si2.info.getDocCount(),
- false, codec, null, null);
+ false, codec, null);
info.setFiles(new HashSet<String>(trackingDir.getCreatedFiles()));
if (useCompoundFile) {
@@ -239,7 +239,7 @@ public class TestDoc extends LuceneTestC
}
}
- return new SegmentInfoPerCommit(info, 0, -1L);
+ return new SegmentInfoPerCommit(info, 0, -1L, -1L);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Fri Nov 1 06:03:52 2013
@@ -50,7 +50,7 @@ public class TestDuelingCodecs extends L
public void setUp() throws Exception {
super.setUp();
- // for now its SimpleText vs Lucene45(random postings format)
+ // for now its SimpleText vs Lucene46(random postings format)
// as this gives the best overall coverage. when we have more
// codecs we should probably pick 2 from Codec.availableCodecs()
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java Fri Nov 1 06:03:52 2013
@@ -17,19 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
+import java.io.IOException;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
-
-import java.io.IOException;
-import java.util.Arrays;
+import org.apache.lucene.util.LuceneTestCase;
//import org.cnlp.utils.properties.ResourceBundleHelper;
@@ -60,14 +57,14 @@ public class TestFieldInfos extends Luce
//Use a RAMOutputStream
FieldInfosWriter writer = Codec.getDefault().fieldInfosFormat().getFieldInfosWriter();
- writer.write(dir, filename, fieldInfos, IOContext.DEFAULT);
+ writer.write(dir, filename, "", fieldInfos, IOContext.DEFAULT);
output.close();
return fieldInfos;
}
public FieldInfos readFieldInfos(Directory dir, String filename) throws IOException {
FieldInfosReader reader = Codec.getDefault().fieldInfosFormat().getFieldInfosReader();
- return reader.read(dir, filename, IOContext.DEFAULT);
+ return reader.read(dir, filename, "", IOContext.DEFAULT);
}
public void test() throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Nov 1 06:03:52 2013
@@ -30,7 +30,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -40,7 +42,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
@@ -1228,192 +1229,4 @@ public class TestIndexWriterDelete exten
r.close();
d.close();
}
-
- // Make sure if we hit a transient IOException (e.g., disk
- // full), and then the exception stops (e.g., disk frees
- // up), so we successfully close IW or open an NRT
- // reader, we don't lose any deletes:
- public void testNoLostDeletesOnIOException() throws Exception {
-
- int deleteCount = 0;
- int docBase = 0;
- int docCount = 0;
-
- MockDirectoryWrapper dir = newMockDirectory();
- final AtomicBoolean shouldFail = new AtomicBoolean();
- dir.failOn(new MockDirectoryWrapper.Failure() {
-
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- StackTraceElement[] trace = new Exception().getStackTrace();
- if (shouldFail.get() == false) {
- return;
- }
-
- boolean sawSeal = false;
- boolean sawWrite = false;
- for (int i = 0; i < trace.length; i++) {
- if ("sealFlushedSegment".equals(trace[i].getMethodName())) {
- sawSeal = true;
- break;
- }
- if ("writeLiveDocs".equals(trace[i].getMethodName())) {
- sawWrite = true;
- }
- }
-
- // Don't throw exc if we are "flushing", else
- // the segment is aborted and docs are lost:
- if (sawWrite && sawSeal == false && random().nextInt(3) == 2) {
- // Only sometimes throw the exc, so we get
- // it sometimes on creating the file, on
- // flushing buffer, on closing the file:
- if (VERBOSE) {
- System.out.println("TEST: now fail; thread=" + Thread.currentThread().getName() + " exc:");
- new Throwable().printStackTrace(System.out);
- }
- shouldFail.set(false);
- throw new FakeIOException();
- }
- }
- });
-
- RandomIndexWriter w = null;
-
- for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {
- int numDocs = atLeast(100);
- if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter + " numDocs=" + numDocs + " docBase=" + docBase + " delCount=" + deleteCount);
- }
- if (w == null) {
- IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- final MergeScheduler ms = iwc.getMergeScheduler();
- if (ms instanceof ConcurrentMergeScheduler) {
- final ConcurrentMergeScheduler suppressFakeIOE = new ConcurrentMergeScheduler() {
- @Override
- protected void handleMergeException(Throwable exc) {
- // suppress only FakeIOException:
- if (!(exc instanceof FakeIOException)) {
- super.handleMergeException(exc);
- }
- }
- };
- final ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) ms;
- suppressFakeIOE.setMaxMergesAndThreads(cms.getMaxMergeCount(), cms.getMaxThreadCount());
- suppressFakeIOE.setMergeThreadPriority(cms.getMergeThreadPriority());
- iwc.setMergeScheduler(suppressFakeIOE);
- }
- w = new RandomIndexWriter(random(), dir, iwc);
- // Since we hit exc during merging, a partial
- // forceMerge can easily return when there are still
- // too many segments in the index:
- w.setDoRandomForceMergeAssert(false);
- }
- for(int i=0;i<numDocs;i++) {
- Document doc = new Document();
- doc.add(new StringField("id", ""+(docBase+i), Field.Store.NO));
- w.addDocument(doc);
- }
- docCount += numDocs;
-
- // TODO: we could make the test more evil, by letting
- // it throw more than one exc, randomly, before "recovering"
-
- // TODO: we could also install an infoStream and try
- // to fail in "more evil" places inside BDS
-
- shouldFail.set(true);
- boolean doClose = false;
-
- try {
-
- for(int i=0;i<numDocs;i++) {
- if (random().nextInt(10) == 7) {
- if (VERBOSE) {
- System.out.println(" delete id=" + (docBase+i));
- }
- deleteCount++;
- w.deleteDocuments(new Term("id", ""+(docBase+i)));
- }
- }
-
- // Trigger writeLiveDocs so we hit fake exc:
- IndexReader r = w.getReader(true);
-
- // Sometimes we will make it here (we only randomly
- // throw the exc):
- assertEquals(docCount-deleteCount, r.numDocs());
- r.close();
-
- // Sometimes close, so the disk full happens on close:
- if (random().nextBoolean()) {
- if (VERBOSE) {
- System.out.println(" now close writer");
- }
- doClose = true;
- w.close();
- w = null;
- }
-
- } catch (FakeIOException ioe) {
- // expected
- if (VERBOSE) {
- System.out.println("TEST: w.close() hit expected IOE");
- }
- // No exception should happen here (we only fail once):
- }
- shouldFail.set(false);
-
- IndexReader r;
-
- if (doClose && w != null) {
- if (VERBOSE) {
- System.out.println(" now 2nd close writer");
- }
- w.close();
- w = null;
- }
-
- if (w == null || random().nextBoolean()) {
- // Open non-NRT reader, to make sure the "on
- // disk" bits are good:
- if (VERBOSE) {
- System.out.println("TEST: verify against non-NRT reader");
- }
- if (w != null) {
- w.commit();
- }
- r = DirectoryReader.open(dir);
- } else {
- if (VERBOSE) {
- System.out.println("TEST: verify against NRT reader");
- }
- r = w.getReader();
- }
- assertEquals(docCount-deleteCount, r.numDocs());
- r.close();
-
- // Sometimes re-use RIW, other times open new one:
- if (w != null && random().nextBoolean()) {
- if (VERBOSE) {
- System.out.println("TEST: close writer");
- }
- w.close();
- w = null;
- }
-
- docBase += numDocs;
- }
-
- if (w != null) {
- w.close();
- }
-
- // Final verify:
- IndexReader r = DirectoryReader.open(dir);
- assertEquals(docCount-deleteCount, r.numDocs());
- r.close();
-
- dir.close();
- }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1537832&r1=1537831&r2=1537832&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Nov 1 06:03:52 2013
@@ -27,7 +27,13 @@ import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -47,6 +53,7 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -1695,7 +1702,230 @@ public class TestIndexWriterExceptions e
dir.close();
}
-
-
+
+ // Make sure if we hit a transient IOException (e.g., disk
+ // full), and then the exception stops (e.g., disk frees
+ // up), so we successfully close IW or open an NRT
+ // reader, we don't lose any deletes or updates:
+ public void testNoLostDeletesOrUpdates() throws Exception {
+ int deleteCount = 0;
+ int docBase = 0;
+ int docCount = 0;
+
+ MockDirectoryWrapper dir = newMockDirectory();
+ final AtomicBoolean shouldFail = new AtomicBoolean();
+ dir.failOn(new MockDirectoryWrapper.Failure() {
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ StackTraceElement[] trace = new Exception().getStackTrace();
+ if (shouldFail.get() == false) {
+ return;
+ }
+
+ boolean sawSeal = false;
+ boolean sawWrite = false;
+ for (int i = 0; i < trace.length; i++) {
+ if ("sealFlushedSegment".equals(trace[i].getMethodName())) {
+ sawSeal = true;
+ break;
+ }
+ if ("writeLiveDocs".equals(trace[i].getMethodName()) || "writeFieldUpdates".equals(trace[i].getMethodName())) {
+ sawWrite = true;
+ }
+ }
+
+ // Don't throw exc if we are "flushing", else
+ // the segment is aborted and docs are lost:
+ if (sawWrite && sawSeal == false && random().nextInt(3) == 2) {
+ // Only sometimes throw the exc, so we get
+ // it sometimes on creating the file, on
+ // flushing buffer, on closing the file:
+ if (VERBOSE) {
+ System.out.println("TEST: now fail; thread=" + Thread.currentThread().getName() + " exc:");
+ new Throwable().printStackTrace(System.out);
+ }
+ shouldFail.set(false);
+ throw new FakeIOException();
+ }
+ }
+ });
+
+ RandomIndexWriter w = null;
+
+ for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {
+ int numDocs = atLeast(100);
+ if (VERBOSE) {
+ System.out.println("\nTEST: iter=" + iter + " numDocs=" + numDocs + " docBase=" + docBase + " delCount=" + deleteCount);
+ }
+ if (w == null) {
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ final MergeScheduler ms = iwc.getMergeScheduler();
+ if (ms instanceof ConcurrentMergeScheduler) {
+ final ConcurrentMergeScheduler suppressFakeIOE = new ConcurrentMergeScheduler() {
+ @Override
+ protected void handleMergeException(Throwable exc) {
+ // suppress only FakeIOException:
+ if (!(exc instanceof FakeIOException)) {
+ super.handleMergeException(exc);
+ }
+ }
+ };
+ final ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) ms;
+ suppressFakeIOE.setMaxMergesAndThreads(cms.getMaxMergeCount(), cms.getMaxThreadCount());
+ suppressFakeIOE.setMergeThreadPriority(cms.getMergeThreadPriority());
+ iwc.setMergeScheduler(suppressFakeIOE);
+ }
+
+ w = new RandomIndexWriter(random(), dir, iwc);
+ // Since we hit exc during merging, a partial
+ // forceMerge can easily return when there are still
+ // too many segments in the index:
+ w.setDoRandomForceMergeAssert(false);
+ }
+ for(int i=0;i<numDocs;i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", ""+(docBase+i), Field.Store.NO));
+ if (defaultCodecSupportsDocValues()) {
+ doc.add(new NumericDocValuesField("f", 1L));
+ doc.add(new NumericDocValuesField("cf", 2L));
+ }
+ w.addDocument(doc);
+ }
+ docCount += numDocs;
+
+ // TODO: we could make the test more evil, by letting
+ // it throw more than one exc, randomly, before "recovering"
+
+ // TODO: we could also install an infoStream and try
+ // to fail in "more evil" places inside BDS
+
+ shouldFail.set(true);
+ boolean doClose = false;
+
+ try {
+
+ boolean defaultCodecSupportsFieldUpdates = defaultCodecSupportsFieldUpdates();
+ for(int i=0;i<numDocs;i++) {
+ if (random().nextInt(10) == 7) {
+ boolean fieldUpdate = defaultCodecSupportsFieldUpdates && random().nextBoolean();
+ if (fieldUpdate) {
+ long value = iter;
+ if (VERBOSE) {
+ System.out.println(" update id=" + (docBase+i) + " to value " + value);
+ }
+ w.updateNumericDocValue(new Term("id", Integer.toString(docBase + i)), "f", value);
+ w.updateNumericDocValue(new Term("id", Integer.toString(docBase + i)), "cf", value * 2);
+ }
+
+ // sometimes do both deletes and updates
+ if (!fieldUpdate || random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println(" delete id=" + (docBase+i));
+ }
+ deleteCount++;
+ w.deleteDocuments(new Term("id", ""+(docBase+i)));
+ }
+ }
+ }
+
+ // Trigger writeLiveDocs so we hit fake exc:
+ IndexReader r = w.getReader(true);
+
+ // Sometimes we will make it here (we only randomly
+ // throw the exc):
+ assertEquals(docCount-deleteCount, r.numDocs());
+ r.close();
+
+ // Sometimes close, so the disk full happens on close:
+ if (random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println(" now close writer");
+ }
+ doClose = true;
+ w.close();
+ w = null;
+ }
+
+ } catch (IOException ioe) {
+ // FakeIOException can be thrown from mergeMiddle, in which case IW
+ // registers it before our CMS gets to suppress it. IW.forceMerge later
+ // throws it as a wrapped IOE, so don't fail in this case.
+ if (ioe instanceof FakeIOException || (ioe.getCause() != null && ioe.getCause() instanceof FakeIOException)) {
+ // expected
+ if (VERBOSE) {
+ System.out.println("TEST: w.close() hit expected IOE");
+ }
+ } else {
+ throw ioe;
+ }
+ }
+ shouldFail.set(false);
+
+ IndexReader r;
+
+ if (doClose && w != null) {
+ if (VERBOSE) {
+ System.out.println(" now 2nd close writer");
+ }
+ w.close();
+ w = null;
+ }
+
+ if (w == null || random().nextBoolean()) {
+ // Open non-NRT reader, to make sure the "on
+ // disk" bits are good:
+ if (VERBOSE) {
+ System.out.println("TEST: verify against non-NRT reader");
+ }
+ if (w != null) {
+ w.commit();
+ }
+ r = DirectoryReader.open(dir);
+ } else {
+ if (VERBOSE) {
+ System.out.println("TEST: verify against NRT reader");
+ }
+ r = w.getReader();
+ }
+ assertEquals(docCount-deleteCount, r.numDocs());
+ if (defaultCodecSupportsDocValues()) {
+ for (AtomicReaderContext context : r.leaves()) {
+ Bits liveDocs = context.reader().getLiveDocs();
+ NumericDocValues f = context.reader().getNumericDocValues("f");
+ NumericDocValues cf = context.reader().getNumericDocValues("cf");
+ for (int i = 0; i < context.reader().maxDoc(); i++) {
+ if (liveDocs == null || liveDocs.get(i)) {
+ assertEquals("doc=" + (docBase + i), cf.get(i), f.get(i) * 2);
+ }
+ }
+ }
+ }
+
+ r.close();
+
+ // Sometimes re-use RIW, other times open new one:
+ if (w != null && random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println("TEST: close writer");
+ }
+ w.close();
+ w = null;
+ }
+
+ docBase += numDocs;
+ }
+
+ if (w != null) {
+ w.close();
+ }
+
+ // Final verify:
+ IndexReader r = DirectoryReader.open(dir);
+ assertEquals(docCount-deleteCount, r.numDocs());
+ r.close();
+
+ dir.close();
+ }
}