You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/09/19 15:24:10 UTC
svn commit: r1626204 [1/2] - in /lucene/dev/branches/lucene_solr_4_10: ./
lucene/ lucene/codecs/
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/ lucene/core/
lucene/core/src/java/org/apache/lucene/codecs/lucene40/
lucene/core/src/java/org/a...
Author: mikemccand
Date: Fri Sep 19 13:24:09 2014
New Revision: 1626204
URL: http://svn.apache.org/r1626204
Log:
LUCENE-5904: backport to 4.10.1
Modified:
lucene/dev/branches/lucene_solr_4_10/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/codecs/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (contents, props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java
lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
lucene/dev/branches/lucene_solr_4_10/lucene/misc/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
lucene/dev/branches/lucene_solr_4_10/lucene/replicator/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java
lucene/dev/branches/lucene_solr_4_10/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
lucene/dev/branches/lucene_solr_4_10/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java
lucene/dev/branches/lucene_solr_4_10/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
lucene/dev/branches/lucene_solr_4_10/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_10/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/lucene_solr_4_10/solr/ (props changed)
lucene/dev/branches/lucene_solr_4_10/solr/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_10/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
lucene/dev/branches/lucene_solr_4_10/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/CHANGES.txt Fri Sep 19 13:24:09 2014
@@ -35,6 +35,16 @@ Tests
* LUCENE-5936: Add backcompat checks to verify what is tested matches known versions
(Ryan Ernst)
+* LUCENE-5904: Fixed a corruption case that can happen when 1)
+ IndexWriter is uncleanly shut-down (OS crash, power loss, etc.), 2)
+ on startup, when a new IndexWriter is created, a virus checker is
+ holding some of the previously written but unused files open and
+ preventing deletion, 3) IndexWriter writes these files again during
+ the course of indexing, then the files can later be deleted, causing
+ corruption. This case was detected by adding evilness to
+ MockDirectoryWrapper to have it simulate a virus checker holding a
+ file open and preventing deletion (Robert Muir, Mike McCandless)
+
======================= Lucene 4.10.0 ======================
New Features
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java Fri Sep 19 13:24:09 2014
@@ -110,10 +110,7 @@ public class SimpleTextSegmentInfoWriter
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(output);
- try {
- dir.deleteFile(segFileName);
- } catch (Throwable t) {
- }
+ IOUtils.deleteFilesIgnoringExceptions(dir, segFileName);
} else {
output.close();
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java Fri Sep 19 13:24:09 2014
@@ -67,7 +67,8 @@ public class Lucene40SegmentInfoWriter e
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(output);
- si.dir.deleteFile(fileName);
+ // TODO: why must we do this? do we not get tracking dir wrapper?
+ IOUtils.deleteFilesIgnoringExceptions(si.dir, fileName);
} else {
output.close();
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java Fri Sep 19 13:24:09 2014
@@ -69,7 +69,8 @@ public class Lucene46SegmentInfoWriter e
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(output);
- si.dir.deleteFile(fileName);
+ // TODO: are we doing this outside of the tracking wrapper? why must SIWriter cleanup like this?
+ IOUtils.deleteFilesIgnoringExceptions(si.dir, fileName);
} else {
output.close();
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Sep 19 13:24:09 2014
@@ -152,7 +152,7 @@ final class IndexFileDeleter implements
// it means the directory is empty, so ignore it.
files = new String[0];
}
-
+
if (currentSegmentsFile != null) {
Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
for (String fileName : files) {
@@ -238,6 +238,9 @@ final class IndexFileDeleter implements
// We keep commits list in sorted order (oldest to newest):
CollectionUtil.timSort(commits);
+ // refCounts only includes "normal" filenames (does not include segments.gen, write.lock)
+ inflateGens(segmentInfos, refCounts.keySet(), infoStream);
+
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
@@ -265,6 +268,83 @@ final class IndexFileDeleter implements
deleteCommits();
}
+ /** Set all gens beyond what we currently see in the directory, to avoid double-write in cases where the previous IndexWriter did not
+ * gracefully close/rollback (e.g. os/machine crashed or lost power). */
+ static void inflateGens(SegmentInfos infos, Collection<String> files, InfoStream infoStream) {
+
+ long maxSegmentGen = Long.MIN_VALUE;
+ int maxSegmentName = Integer.MIN_VALUE;
+
+ // Confusingly, this is the union of liveDocs, field infos, doc values
+ // (and maybe others, in the future) gens. This is somewhat messy,
+ // since it means DV updates will suddenly write to the next gen after
+ // live docs' gen, for example, but we don't have the APIs to ask the
+ // codec which file is which:
+ Map<String,Long> maxPerSegmentGen = new HashMap<>();
+
+ for(String fileName : files) {
+ if (fileName.equals(IndexFileNames.SEGMENTS_GEN) || fileName.equals(IndexWriter.WRITE_LOCK_NAME)) {
+ // do nothing
+ } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+ try {
+ maxSegmentGen = Math.max(SegmentInfos.generationFromSegmentsFileName(fileName), maxSegmentGen);
+ } catch (NumberFormatException ignore) {
+ // trash file: we have to handle this since we allow anything starting with 'segments' here
+ }
+ } else {
+ String segmentName = IndexFileNames.parseSegmentName(fileName);
+ assert segmentName.startsWith("_"): "wtf? file=" + fileName;
+
+ maxSegmentName = Math.max(maxSegmentName, Integer.parseInt(segmentName.substring(1), Character.MAX_RADIX));
+
+ Long curGen = maxPerSegmentGen.get(segmentName);
+ if (curGen == null) {
+ curGen = 0L;
+ }
+
+ try {
+ curGen = Math.max(curGen, IndexFileNames.parseGeneration(fileName));
+ } catch (NumberFormatException ignore) {
+ // trash file: we have to handle this since codec regex is only so good
+ }
+ maxPerSegmentGen.put(segmentName, curGen);
+ }
+ }
+
+ // Generation is advanced before write:
+ infos.setGeneration(Math.max(infos.getGeneration(), maxSegmentGen));
+ if (infos.counter < 1+maxSegmentName) {
+ if (infoStream.isEnabled("IFD")) {
+ infoStream.message("IFD", "init: inflate infos.counter to " + (1+maxSegmentName) + " vs current=" + infos.counter);
+ }
+ infos.counter = 1+maxSegmentName;
+ }
+
+ for(SegmentCommitInfo info : infos) {
+ Long gen = maxPerSegmentGen.get(info.info.name);
+ assert gen != null;
+ long genLong = gen;
+ if (info.getNextWriteDelGen() < genLong+1) {
+ if (infoStream.isEnabled("IFD")) {
+ infoStream.message("IFD", "init: seg=" + info.info.name + " set nextWriteDelGen=" + (genLong+1) + " vs current=" + info.getNextWriteDelGen());
+ }
+ info.setNextWriteDelGen(genLong+1);
+ }
+ if (info.getNextWriteFieldInfosGen() < genLong+1) {
+ if (infoStream.isEnabled("IFD")) {
+ infoStream.message("IFD", "init: seg=" + info.info.name + " set nextWriteFieldInfosGen=" + (genLong+1) + " vs current=" + info.getNextWriteFieldInfosGen());
+ }
+ info.setNextWriteFieldInfosGen(genLong+1);
+ }
+ if (info.getNextWriteDocValuesGen() < genLong+1) {
+ if (infoStream.isEnabled("IFD")) {
+ infoStream.message("IFD", "init: seg=" + info.info.name + " set nextWriteDocValuesGen=" + (genLong+1) + " vs current=" + info.getNextWriteDocValuesGen());
+ }
+ info.setNextWriteDocValuesGen(genLong+1);
+ }
+ }
+ }
+
private void ensureOpen() throws AlreadyClosedException {
writer.ensureOpen(false);
// since we allow 'closing' state, we must still check this, we could be closing because we hit e.g. OOM
@@ -424,10 +504,17 @@ final class IndexFileDeleter implements
deletable = null;
int size = oldDeletable.size();
for(int i=0;i<size;i++) {
+ String fileName = oldDeletable.get(i);
if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "delete pending file " + oldDeletable.get(i));
+ infoStream.message("IFD", "delete pending file " + fileName);
+ }
+ RefCount rc = refCounts.get(fileName);
+ if (rc != null && rc.count > 0) {
+ // LUCENE-5904: should never happen! This means we are about to pending-delete a referenced index file
+ assert false: "fileName=" + fileName + " is in pending delete list but also has refCount=" + rc.count;
+ } else {
+ deleteFile(fileName);
}
- deleteFile(oldDeletable.get(i));
}
}
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java Fri Sep 19 13:24:09 2014
@@ -172,6 +172,23 @@ public final class IndexFileNames {
}
return filename;
}
+
+ /** Returns the generation from this file name, or 0 if there is no
+ * generation. */
+ public static long parseGeneration(String filename) {
+ assert filename.startsWith("_");
+ String parts[] = stripExtension(filename).substring(1).split("_");
+ // 4 cases:
+ // segment.ext
+ // segment_gen.ext
+ // segment_codec_suffix.ext
+ // segment_gen_codec_suffix.ext
+ if (parts.length == 2 || parts.length == 4) {
+ return Long.parseLong(parts[1], Character.MAX_RADIX);
+ } else {
+ return 0;
+ }
+ }
/**
* Parses the segment name out of the given file name.
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Fri Sep 19 13:24:09 2014
@@ -192,11 +192,7 @@ public class PersistentSnapshotDeletionP
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(out);
- try {
- dir.deleteFile(fileName);
- } catch (Exception e) {
- // Suppress so we keep throwing original exception
- }
+ IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
} else {
IOUtils.close(out);
}
@@ -206,11 +202,8 @@ public class PersistentSnapshotDeletionP
if (nextWriteGen > 0) {
String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
- try {
- dir.deleteFile(lastSaveFile);
- } catch (IOException ioe) {
- // OK: likely it didn't exist
- }
+ // exception OK: likely it didn't exist
+ IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
}
nextWriteGen++;
@@ -287,7 +280,7 @@ public class PersistentSnapshotDeletionP
String curFileName = SNAPSHOTS_PREFIX + genLoaded;
for(String file : snapshotFiles) {
if (!curFileName.equals(file)) {
- dir.deleteFile(file);
+ IOUtils.deleteFilesIgnoringExceptions(dir, file);
}
}
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Fri Sep 19 13:24:09 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.MutableBits;
// Used by IndexWriter to hold open SegmentReaders (for
@@ -331,11 +332,7 @@ class ReadersAndUpdates {
// Delete any partially created file(s):
for (String fileName : trackingDir.getCreatedFiles()) {
- try {
- dir.deleteFile(fileName);
- } catch (Throwable t) {
- // Ignore so we throw only the first exc
- }
+ IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
}
}
}
@@ -589,11 +586,7 @@ class ReadersAndUpdates {
// Delete any partially created file(s):
for (String fileName : trackingDir.getCreatedFiles()) {
- try {
- dir.deleteFile(fileName);
- } catch (Throwable t) {
- // Ignore so we throw only the first exc
- }
+ IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
}
}
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java Fri Sep 19 13:24:09 2014
@@ -148,6 +148,16 @@ public class SegmentCommitInfo {
nextWriteDelGen++;
}
+ /** Gets the nextWriteDelGen. */
+ long getNextWriteDelGen() {
+ return nextWriteDelGen;
+ }
+
+ /** Sets the nextWriteDelGen. */
+ void setNextWriteDelGen(long v) {
+ nextWriteDelGen = v;
+ }
+
/** Called when we succeed in writing a new FieldInfos generation. */
void advanceFieldInfosGen() {
fieldInfosGen = nextWriteFieldInfosGen;
@@ -163,6 +173,16 @@ public class SegmentCommitInfo {
nextWriteFieldInfosGen++;
}
+ /** Gets the nextWriteFieldInfosGen. */
+ long getNextWriteFieldInfosGen() {
+ return nextWriteFieldInfosGen;
+ }
+
+ /** Sets the nextWriteFieldInfosGen. */
+ void setNextWriteFieldInfosGen(long v) {
+ nextWriteFieldInfosGen = v;
+ }
+
/** Called when we succeed in writing a new DocValues generation. */
void advanceDocValuesGen() {
docValuesGen = nextWriteDocValuesGen;
@@ -178,6 +198,16 @@ public class SegmentCommitInfo {
nextWriteDocValuesGen++;
}
+ /** Gets the nextWriteDocValuesGen. */
+ long getNextWriteDocValuesGen() {
+ return nextWriteDocValuesGen;
+ }
+
+ /** Sets the nextWriteDocValuesGen. */
+ void setNextWriteDocValuesGen(long v) {
+ nextWriteDocValuesGen = v;
+ }
+
/** Returns total size in bytes of all files for this
* segment.
* <p><b>NOTE:</b> This value is not correct for 3.0 segments
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Fri Sep 19 13:24:09 2014
@@ -152,6 +152,7 @@ public final class SegmentInfos implemen
public static final int FORMAT_SEGMENTS_GEN_CURRENT = FORMAT_SEGMENTS_GEN_CHECKSUM;
/** Used to name new segments. */
+ // TODO: should this be a long ...?
public int counter;
/** Counts how often the index has been changed. */
@@ -298,12 +299,7 @@ public final class SegmentInfos implemen
} catch (Throwable t) {
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
- try {
- dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
- } catch (Throwable t2) {
- // Ignore; this file is only used in a retry
- // fallback on init.
- }
+ IOUtils.deleteFilesIgnoringExceptions(dir, IndexFileNames.SEGMENTS_GEN);
}
}
@@ -556,21 +552,13 @@ public final class SegmentInfos implemen
// but suppress any exception:
IOUtils.closeWhileHandlingException(segnOutput);
- for(String fileName : upgradedSIFiles) {
- try {
- directory.deleteFile(fileName);
- } catch (Throwable t) {
- // Suppress so we keep throwing the original exception
- }
+ for (String fileName : upgradedSIFiles) {
+ IOUtils.deleteFilesIgnoringExceptions(directory, fileName);
}
- try {
- // Try not to leave a truncated segments_N file in
- // the index:
- directory.deleteFile(segmentsFileName);
- } catch (Throwable t) {
- // Suppress so we keep throwing the original exception
- }
+ // Try not to leave a truncated segments_N file in
+ // the index:
+ IOUtils.deleteFilesIgnoringExceptions(directory, segmentsFileName);
}
}
}
@@ -750,7 +738,7 @@ public final class SegmentInfos implemen
* commit finishing.
*/
public abstract static class FindSegmentsFile {
-
+
final Directory directory;
/** Sole constructor. */
@@ -922,6 +910,9 @@ public final class SegmentInfos implemen
return v;
} catch (IOException err) {
+ // TODO: we should use the new IO apis in Java7 to get better exceptions on why the open failed. E.g. we don't want to fall back
+ // if the open failed for a "different" reason (too many open files, access denied) than "the commit was in progress"
+
// Save the original root cause:
if (exc == null) {
exc = err;
@@ -987,6 +978,11 @@ public final class SegmentInfos implemen
generation = other.generation;
}
+ void setGeneration(long generation) {
+ this.generation = generation;
+ this.lastGeneration = generation;
+ }
+
final void rollbackCommit(Directory dir) {
if (pendingSegnOutput != null) {
// Suppress so we keep throwing the original exception
@@ -1092,11 +1088,7 @@ public final class SegmentInfos implemen
success = true;
} finally {
if (!success) {
- try {
- dir.deleteFile(fileName);
- } catch (Throwable t) {
- // Suppress so we keep throwing the original exception
- }
+ IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
}
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java Fri Sep 19 13:24:09 2014
@@ -65,7 +65,9 @@ public class SegmentWriteState {
* each of the postings formats it wraps. If you create
* a new {@link PostingsFormat} then any files you
* write/read must be derived using this suffix (use
- * {@link IndexFileNames#segmentFileName(String,String,String)}). */
+ * {@link IndexFileNames#segmentFileName(String,String,String)}).
+ *
+ * Note: the suffix must be either empty, or be a textual suffix contain exactly two parts (separated by underscore), or be a base36 generation. */
public final String segmentSuffix;
/** Expert: The fraction of terms in the "dictionary" which should be stored
@@ -99,6 +101,7 @@ public class SegmentWriteState {
this.segmentInfo = segmentInfo;
this.fieldInfos = fieldInfos;
this.termIndexInterval = termIndexInterval;
+ assert assertSegmentSuffix(segmentSuffix);
this.segmentSuffix = segmentSuffix;
this.context = context;
}
@@ -115,4 +118,23 @@ public class SegmentWriteState {
segUpdates = state.segUpdates;
delCountOnFlush = state.delCountOnFlush;
}
+
+ // currently only used by assert? clean up and make real check?
+ // either its a segment suffix (_X_Y) or its a parseable generation
+ // TODO: this is very confusing how ReadersAndUpdates passes generations via
+ // this mechanism, maybe add 'generation' explicitly to ctor create the 'actual suffix' here?
+ private boolean assertSegmentSuffix(String segmentSuffix) {
+ assert segmentSuffix != null;
+ if (!segmentSuffix.isEmpty()) {
+ int numParts = segmentSuffix.split("_").length;
+ if (numParts == 2) {
+ return true;
+ } else if (numParts == 1) {
+ Long.parseLong(segmentSuffix, Character.MAX_RADIX);
+ return true;
+ }
+ return false; // invalid
+ }
+ return true;
+ }
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Fri Sep 19 13:24:09 2014
@@ -203,7 +203,8 @@ final class CompoundFileWriter implement
if (success) {
IOUtils.close(is);
// copy successful - delete file
- fileEntry.dir.deleteFile(fileEntry.file);
+ // if we can't we rely on IFD to pick up and retry
+ IOUtils.deleteFilesIgnoringExceptions(fileEntry.dir, fileEntry.file);
} else {
IOUtils.closeWhileHandlingException(is);
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java Fri Sep 19 13:24:09 2014
@@ -30,6 +30,7 @@ import org.apache.lucene.index.BaseStore
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
@@ -46,6 +47,10 @@ public class TestCompressingStoredFields
@Test(expected=IllegalArgumentException.class)
public void testDeletePartiallyWrittenFilesIfAbort() throws IOException {
Directory dir = newDirectory();
+ // test explicitly needs files to always be actually deleted
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
iwConf.setCodec(CompressingCodec.randomInstance(random()));
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Sep 19 13:24:09 2014
@@ -1093,7 +1093,8 @@ public class TestAddIndexes extends Luce
IndexReader[] readers = new IndexReader[] { DirectoryReader.open(dirs[0]), DirectoryReader.open(dirs[1]) };
- Directory dir = new MockDirectoryWrapper(random(), new RAMDirectory());
+ MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
+ dir.setEnableVirusScanner(false); // we check for specific list of files
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(true));
MergePolicy lmp = conf.getMergePolicy();
// Force creation of CFS:
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java Fri Sep 19 13:24:09 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.document.Numeri
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -38,6 +39,10 @@ import org.apache.lucene.util.TestUtil;
public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(new Lucene410Codec());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Fri Sep 19 13:24:09 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.document.Numeri
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -38,6 +39,12 @@ import org.apache.lucene.util.TestUtil;
public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
+
+ if (dir instanceof MockDirectoryWrapper) {
+ // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(new Lucene410Codec());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Fri Sep 19 13:24:09 2014
@@ -58,6 +58,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
@@ -1146,6 +1147,11 @@ public class TestBackwardsCompatibility
System.out.println("testUpgradeOldSingleSegmentIndexWithAdditions: index=" +name);
}
Directory dir = newDirectory(oldIndexDirs.get(name));
+ if (dir instanceof MockDirectoryWrapper) {
+ // we need to ensure we delete old commits for this test,
+ // otherwise IndexUpgrader gets angry
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));
@@ -1177,6 +1183,8 @@ public class TestBackwardsCompatibility
// determine count of segments in modified index
final int origSegCount = getNumberOfSegments(dir);
+ // ensure there is only one commit
+ assertEquals(1, DirectoryReader.listCommits(dir).size());
newIndexUpgrader(dir).upgrade();
final int segCount = checkAllSegmentsUpgraded(dir);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java Fri Sep 19 13:24:09 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Sorted
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -1163,6 +1164,10 @@ public class TestBinaryDocValuesUpdates
public void testDeleteUnusedUpdatesFiles() throws Exception {
Directory dir = newDirectory();
+ // test explicitly needs files to always be actually deleted
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java Fri Sep 19 13:24:09 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -28,7 +29,9 @@ import org.apache.lucene.util.TestUtil;
public class TestCodecHoldsOpenFiles extends LuceneTestCase {
public void test() throws Exception {
- Directory d = newDirectory();
+ BaseDirectoryWrapper d = newDirectory();
+ d.setCheckIndexOnClose(false);
+ // we nuke files, but verify the reader still works
RandomIndexWriter w = new RandomIndexWriter(random(), d);
int numDocs = atLeast(100);
for(int i=0;i<numDocs;i++) {
@@ -38,11 +41,13 @@ public class TestCodecHoldsOpenFiles ext
}
IndexReader r = w.getReader();
+ w.commit();
w.close();
for(String fileName : d.listAll()) {
try {
d.deleteFile(fileName);
+ // may succeed, e.g. if the file is completely read into RAM.
} catch (IOException ioe) {
// ignore: this means codec (correctly) is holding
// the file open
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java Fri Sep 19 13:24:09 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -709,6 +710,10 @@ public class TestCompoundFile extends Lu
public void testReadNestedCFP() throws IOException {
Directory newDir = newDirectory();
+ // manually manipulates directory
+ if (newDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)newDir).setEnableVirusScanner(false);
+ }
CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
CompoundFileDirectory nested = new CompoundFileDirectory(newDir, "b.cfs", newIOContext(random()), true);
IndexOutput out = nested.createOutput("b.xyz", newIOContext(random()));
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Fri Sep 19 13:24:09 2014
@@ -177,6 +177,10 @@ public class TestConcurrentMergeSchedule
public void testNoExtraFiles() throws IOException {
Directory directory = newDirectory();
+ if (directory instanceof MockDirectoryWrapper) {
+ // test uses IW unref'ed helper which is unaware of retries
+ ((MockDirectoryWrapper)directory).setEnableVirusScanner(false);
+ }
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2));
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java Fri Sep 19 13:24:09 2014
@@ -34,6 +34,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -221,6 +222,10 @@ public class TestDeletionPolicy extends
final double SECONDS = 2.0;
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // test manually deletes files
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
.setIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(dir, SECONDS));
MergePolicy mp = conf.getMergePolicy();
@@ -314,6 +319,10 @@ public class TestDeletionPolicy extends
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // test manually deletes files
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
.setIndexDeletionPolicy(new KeepAllDeletionPolicy(dir))
@@ -562,6 +571,10 @@ public class TestDeletionPolicy extends
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // test manually deletes files
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
for(int j=0;j<N+1;j++) {
@@ -623,6 +636,10 @@ public class TestDeletionPolicy extends
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // test manually deletes files
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE)
.setIndexDeletionPolicy(new KeepLastNDeletionPolicy(N))
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java Fri Sep 19 13:24:09 2014
@@ -629,6 +629,10 @@ public class TestDirectoryReaderReopen e
public void testOverDecRefDuringReopen() throws Exception {
MockDirectoryWrapper dir = newMockDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // ensure we produce enough of our exceptions
+ dir.setEnableVirusScanner(false);
+ }
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setCodec(Codec.forName("Lucene410"));
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Fri Sep 19 13:24:09 2014
@@ -119,6 +119,8 @@ public class TestDoc extends LuceneTestC
// We create unreferenced files (we don't even write
// a segments file):
((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+ // this test itself deletes files (has no retry mechanism)
+ ((MockDirectoryWrapper) directory).setEnableVirusScanner(false);
}
IndexWriter writer = new IndexWriter(
@@ -161,6 +163,8 @@ public class TestDoc extends LuceneTestC
// We create unreferenced files (we don't even write
// a segments file):
((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+ // this test itself deletes files (has no retry mechanism)
+ ((MockDirectoryWrapper) directory).setEnableVirusScanner(false);
}
writer = new IndexWriter(
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Fri Sep 19 13:24:09 2014
@@ -28,9 +28,11 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
/*
@@ -44,6 +46,8 @@ public class TestIndexFileDeleter extend
Directory dir = newDirectory();
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
+ // ensure we actually delete files
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
}
MergePolicy mergePolicy = newLogMergePolicy(true, 10);
@@ -298,4 +302,194 @@ public class TestIndexFileDeleter extend
w.close();
dir.close();
}
+
+ public void testVirusScannerDoesntCorruptIndex() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+ dir.setPreventDoubleWrite(false); // we arent trying to test this
+ dir.setEnableVirusScanner(false); // we have our own to make test reproduce always
+
+ // add empty commit
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+ // add a trash unreferenced file
+ dir.createOutput("_0.si", IOContext.DEFAULT).close();
+
+ // start virus scanner
+ final AtomicBoolean stopScanning = new AtomicBoolean();
+ dir.failOn(new MockDirectoryWrapper.Failure() {
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (stopScanning.get()) {
+ return;
+ }
+ for (StackTraceElement f : new Exception().getStackTrace()) {
+ if ("deleteFile".equals(f.getMethodName())) {
+ throw new IOException("temporarily cannot delete file");
+ }
+ }
+ }
+ });
+
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+ iw.addDocument(new Document());
+ // stop virus scanner
+ stopScanning.set(true);
+ iw.commit();
+ iw.close();
+ dir.close();
+ }
+
+ public void testNoSegmentsDotGenInflation() throws IOException {
+ Directory dir = newMockDirectory();
+
+ // empty commit
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.getGeneration());
+
+ // no inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(1, sis.getGeneration());
+
+ dir.close();
+ }
+
+ public void testSegmentsInflation() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+ dir.setCheckIndexOnClose(false); // TODO: allow falling back more than one commit
+
+ // empty commit
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.getGeneration());
+
+ // add trash commit
+ dir.createOutput(IndexFileNames.SEGMENTS + "_2", IOContext.DEFAULT).close();
+
+ // ensure inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(2, sis.getGeneration());
+
+ // add another trash commit
+ dir.createOutput(IndexFileNames.SEGMENTS + "_4", IOContext.DEFAULT).close();
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(4, sis.getGeneration());
+
+ dir.close();
+ }
+
+ public void testSegmentNameInflation() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+
+ // empty commit
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(0, sis.counter);
+
+ // no inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(0, sis.counter);
+
+ // add trash per-segment file
+ dir.createOutput(IndexFileNames.segmentFileName("_0", "", "foo"), IOContext.DEFAULT).close();
+
+ // ensure inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(1, sis.counter);
+
+ // add trash per-segment file
+ dir.createOutput(IndexFileNames.segmentFileName("_3", "", "foo"), IOContext.DEFAULT).close();
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(4, sis.counter);
+
+ // ensure we write _4 segment next
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+ iw.addDocument(new Document());
+ iw.commit();
+ iw.close();
+ sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals("_4", sis.info(0).info.name);
+ assertEquals(5, sis.counter);
+
+ dir.close();
+ }
+
+ public void testGenerationInflation() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+
+ // initial commit
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+ iw.addDocument(new Document());
+ iw.commit();
+ iw.close();
+
+ // no deletes: start at 1
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.info(0).getNextDelGen());
+
+ // no inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(1, sis.info(0).getNextDelGen());
+
+ // add trash per-segment deletes file
+ dir.createOutput(IndexFileNames.fileNameFromGeneration("_0", "del", 2), IOContext.DEFAULT).close();
+
+ // ensure inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(3, sis.info(0).getNextDelGen());
+
+ dir.close();
+ }
+
+ public void testTrashyFile() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+ dir.setCheckIndexOnClose(false); // TODO: maybe handle such trash better elsewhere...
+
+ // empty commit
+ new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
+
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.getGeneration());
+
+ // add trash file
+ dir.createOutput(IndexFileNames.SEGMENTS + "_", IOContext.DEFAULT).close();
+
+ // no inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(1, sis.getGeneration());
+
+ dir.close();
+ }
+
+ public void testTrashyGenFile() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+
+ // initial commit
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+ iw.addDocument(new Document());
+ iw.commit();
+ iw.close();
+
+ // no deletes: start at 1
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(dir);
+ assertEquals(1, sis.info(0).getNextDelGen());
+
+ // add trash file
+ dir.createOutput("_1_A", IOContext.DEFAULT).close();
+
+ // no inflation
+ IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+ assertEquals(1, sis.info(0).getNextDelGen());
+
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Sep 19 13:24:09 2014
@@ -171,7 +171,12 @@ public class TestIndexWriter extends Luc
+ // TODO: we have the logic in MDW to do this check, and its better, because it knows about files it tried
+ // to delete but couldn't: we should replace this!!!!
public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
+ if (dir instanceof MockDirectoryWrapper) {
+ assertFalse("test is broken: should disable virus scanner", ((MockDirectoryWrapper)dir).getEnableVirusScanner());
+ }
String[] startFiles = dir.listAll();
new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))).rollback();
String[] endFiles = dir.listAll();
@@ -309,21 +314,31 @@ public class TestIndexWriter extends Luc
.setRAMBufferSizeMB(0.000001)
.setMergePolicy(newLogMergePolicy(10))
);
- int lastNumFile = dir.listAll().length;
+ int lastNumSegments = getSegmentCount(dir);
for(int j=0;j<9;j++) {
Document doc = new Document();
doc.add(newField("field", "aaa" + j, storedTextType));
writer.addDocument(doc);
- int numFile = dir.listAll().length;
// Verify that with a tiny RAM buffer we see new
// segment after every doc
- assertTrue(numFile > lastNumFile);
- lastNumFile = numFile;
+ int numSegments = getSegmentCount(dir);
+ assertTrue(numSegments > lastNumSegments);
+ lastNumSegments = numSegments;
}
writer.close();
dir.close();
}
+ /** Returns how many unique segment names are in the directory. */
+ private static int getSegmentCount(Directory dir) throws IOException {
+ Set<String> segments = new HashSet<>();
+ for(String file : dir.listAll()) {
+ segments.add(IndexFileNames.parseSegmentName(file));
+ }
+
+ return segments.size();
+ }
+
// Make sure it's OK to change RAM buffer size and
// maxBufferedDocs in a write session
public void testChangingRAMBuffer() throws IOException {
@@ -1009,8 +1024,10 @@ public class TestIndexWriter extends Luc
final Directory adder;
final ByteArrayOutputStream bytesLog = new ByteArrayOutputStream();
final PrintStream log = new PrintStream(bytesLog, true, IOUtils.UTF_8);
-
- IndexerThreadInterrupt() throws IOException {
+ final int id;
+
+ IndexerThreadInterrupt(int id) throws IOException {
+ this.id = id;
this.random = new Random(random().nextLong());
// make a little directory for addIndexes
// LUCENE-2239: won't work with NIOFS/MMAP
@@ -1159,7 +1176,7 @@ public class TestIndexWriter extends Luc
// on!! This test doesn't repro easily so when
// Jenkins hits a fail we need to study where the
// interrupts struck!
- log.println("TEST: got interrupt");
+ log.println("TEST thread " + id + ": got interrupt");
re.printStackTrace(log);
Throwable e = re.getCause();
assertTrue(e instanceof InterruptedException);
@@ -1167,7 +1184,7 @@ public class TestIndexWriter extends Luc
break;
}
} catch (Throwable t) {
- log.println("FAILED; unexpected exception");
+ log.println("thread " + id + " FAILED; unexpected exception");
t.printStackTrace(log);
failed = true;
break;
@@ -1175,11 +1192,11 @@ public class TestIndexWriter extends Luc
}
if (VERBOSE) {
- log.println("TEST: now finish failed=" + failed);
+ log.println("TEST: thread " + id + ": now finish failed=" + failed);
}
if (!failed) {
if (VERBOSE) {
- log.println("TEST: now rollback");
+ log.println("TEST: thread " + id + ": now rollback");
}
// clear interrupt state:
Thread.interrupted();
@@ -1195,7 +1212,7 @@ public class TestIndexWriter extends Luc
TestUtil.checkIndex(dir);
} catch (Exception e) {
failed = true;
- log.println("CheckIndex FAILED: unexpected exception");
+ log.println("thread " + id + ": CheckIndex FAILED: unexpected exception");
e.printStackTrace(log);
}
try {
@@ -1204,25 +1221,27 @@ public class TestIndexWriter extends Luc
r.close();
} catch (Exception e) {
failed = true;
- log.println("DirectoryReader.open FAILED: unexpected exception");
+ log.println("thread " + id + ": DirectoryReader.open FAILED: unexpected exception");
e.printStackTrace(log);
}
}
try {
IOUtils.close(dir);
} catch (IOException e) {
- throw new RuntimeException(e);
+ failed = true;
+ throw new RuntimeException("thread " + id, e);
}
try {
IOUtils.close(adder);
} catch (IOException e) {
- throw new RuntimeException(e);
+ failed = true;
+ throw new RuntimeException("thread " + id, e);
}
}
}
public void testThreadInterruptDeadlock() throws Exception {
- IndexerThreadInterrupt t = new IndexerThreadInterrupt();
+ IndexerThreadInterrupt t = new IndexerThreadInterrupt(1);
t.setDaemon(true);
t.start();
@@ -1256,11 +1275,11 @@ public class TestIndexWriter extends Luc
/** testThreadInterruptDeadlock but with 2 indexer threads */
public void testTwoThreadsInterruptDeadlock() throws Exception {
- IndexerThreadInterrupt t1 = new IndexerThreadInterrupt();
+ IndexerThreadInterrupt t1 = new IndexerThreadInterrupt(1);
t1.setDaemon(true);
t1.start();
- IndexerThreadInterrupt t2 = new IndexerThreadInterrupt();
+ IndexerThreadInterrupt t2 = new IndexerThreadInterrupt(2);
t2.setDaemon(true);
t2.start();
@@ -1290,8 +1309,13 @@ public class TestIndexWriter extends Luc
t2.finish = true;
t1.join();
t2.join();
- assertFalse(t1.failed);
- assertFalse(t2.failed);
+ if (t1.failed) {
+ System.out.println("Thread1 failed:\n" + new String(t1.bytesLog.toString("UTF-8")));
+ }
+ if (t2.failed) {
+ System.out.println("Thread2 failed:\n" + new String(t2.bytesLog.toString("UTF-8")));
+ }
+ assertFalse(t1.failed || t2.failed);
}
@@ -1406,7 +1430,8 @@ public class TestIndexWriter extends Luc
public void testDeleteUnusedFiles() throws Exception {
for(int iter=0;iter<2;iter++) {
- Directory dir = newMockDirectory(); // relies on windows semantics
+ MockDirectoryWrapper dir = newMockDirectory(); // relies on windows semantics
+ dir.setEnableVirusScanner(false); // but ensures files are actually deleted
MergePolicy mergePolicy = newLogMergePolicy(true);
@@ -1495,10 +1520,14 @@ public class TestIndexWriter extends Luc
}
}
- public void testDeleteUnsedFiles2() throws Exception {
+ public void testDeleteUnusedFiles2() throws Exception {
// Validates that iw.deleteUnusedFiles() also deletes unused index commits
// in case a deletion policy which holds onto commits is used.
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // otherwise the delete of old commit might not actually succeed temporarily.
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
@@ -1551,6 +1580,9 @@ public class TestIndexWriter extends Luc
// indexed, flushed (but not committed) and then IW rolls back, then no
// files are left in the Directory.
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2)
.setMergePolicy(newLogMergePolicy())
@@ -1637,6 +1669,10 @@ public class TestIndexWriter extends Luc
public void testNoUnwantedTVFiles() throws Exception {
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // test uses IW unref'ed check which is unaware of retries
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setRAMBufferSizeMB(0.01)
.setMergePolicy(newLogMergePolicy()));
@@ -1822,7 +1858,8 @@ public class TestIndexWriter extends Luc
public void testDeleteAllNRTLeftoverFiles() throws Exception {
- Directory d = new MockDirectoryWrapper(random(), new RAMDirectory());
+ MockDirectoryWrapper d = new MockDirectoryWrapper(random(), new RAMDirectory());
+ d.setEnableVirusScanner(false); // needs for files to actually be deleted
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document doc = new Document();
for(int i = 0; i < 20; i++) {
@@ -1947,6 +1984,10 @@ public class TestIndexWriter extends Luc
// LUCENE-3872
public void testPrepareCommitThenRollback() throws Exception {
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // indexExists might return true if virus scanner prevents deletions
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter w = new IndexWriter(dir,
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -2696,6 +2737,9 @@ public class TestIndexWriter extends Luc
assumeFalse("this test can't run on Windows", Constants.WINDOWS);
MockDirectoryWrapper dir = newMockDirectory();
+
+ // don't act like windows either, or the test won't simulate the condition
+ dir.setEnableVirusScanner(false);
// Allow deletion of still open files:
dir.setNoDeleteOpenFile(false);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Fri Sep 19 13:24:09 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -94,6 +95,10 @@ public class TestIndexWriterCommit exten
*/
public void testCommitOnCloseAbort() throws IOException {
Directory dir = newDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // test uses IW unref'ed check which is unaware of retries
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(10));
for (int i = 0; i < 14; i++) {
@@ -185,6 +190,11 @@ public class TestIndexWriterCommit exten
final String contentFormat = TestUtil.getPostingsFormat("content");
assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory"));
MockDirectoryWrapper dir = newMockDirectory();
+ if (dir instanceof MockDirectoryWrapper) {
+ // the virus scanner can use up too much disk space :)
+ // an alternative is to expose MDW.triedToDelete and discount it
+ dir.setEnableVirusScanner(false);
+ }
Analyzer analyzer;
if (random().nextBoolean()) {
// no payloads
@@ -270,6 +280,10 @@ public class TestIndexWriterCommit exten
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
}
+ if (dir instanceof MockDirectoryWrapper) {
+ // test uses IW unref'ed check which is unaware of retries
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter writer = new IndexWriter(
dir,
newIndexWriterConfig(new MockAnalyzer(random()))
@@ -557,8 +571,13 @@ public class TestIndexWriterCommit exten
// LUCENE-1274: test writer.prepareCommit()
public void testPrepareCommitRollback() throws IOException {
Directory dir = newDirectory();
+
+ MockDirectoryWrapper mockDir;
if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
+ mockDir = (MockDirectoryWrapper) dir;
+ mockDir.setPreventDoubleWrite(false);
+ } else {
+ mockDir = null;
}
IndexWriter writer = new IndexWriter(
@@ -569,8 +588,9 @@ public class TestIndexWriterCommit exten
);
writer.commit();
- for (int i = 0; i < 23; i++)
+ for (int i = 0; i < 23; i++) {
TestIndexWriter.addDoc(writer);
+ }
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(0, reader.numDocs());
@@ -580,7 +600,14 @@ public class TestIndexWriterCommit exten
IndexReader reader2 = DirectoryReader.open(dir);
assertEquals(0, reader2.numDocs());
+ // We need to let IW delete the partial segments_N that was written in prepareCommit, else we get a false fail below:
+ if (mockDir != null) {
+ mockDir.setEnableVirusScanner(false);
+ }
writer.rollback();
+ if (mockDir != null) {
+ mockDir.setEnableVirusScanner(true);
+ }
IndexReader reader3 = DirectoryReader.openIfChanged(reader);
assertNull(reader3);
@@ -589,9 +616,12 @@ public class TestIndexWriterCommit exten
reader.close();
reader2.close();
+ // System.out.println("TEST: after rollback: " + Arrays.toString(dir.listAll()));
+
writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- for (int i = 0; i < 17; i++)
+ for (int i = 0; i < 17; i++) {
TestIndexWriter.addDoc(writer);
+ }
reader = DirectoryReader.open(dir);
assertEquals(0, reader.numDocs());
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Sep 19 13:24:09 2014
@@ -508,6 +508,8 @@ public class TestIndexWriterDelete exten
MockDirectoryWrapper startDir = newMockDirectory();
// TODO: find the resource leak that only occurs sometimes here.
startDir.setNoDeleteOpenFile(false);
+ // test uses IW unref'ed helper which is unaware of retries
+ startDir.setEnableVirusScanner(false);
IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
for (int i = 0; i < 157; i++) {
Document d = new Document();
@@ -535,6 +537,8 @@ public class TestIndexWriterDelete exten
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory(startDir, newIOContext(random())));
dir.setPreventDoubleWrite(false);
dir.setAllowRandomFileNotFoundException(false);
+ // test uses IW unref'ed helper which is unaware of retries
+ dir.setEnableVirusScanner(false);
IndexWriter modifier = new IndexWriter(dir,
newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(1000)
@@ -751,7 +755,7 @@ public class TestIndexWriterDelete exten
System.out.println("TEST: mock failure: now fail");
new Throwable().printStackTrace(System.out);
}
- throw new IOException("fail after applyDeletes");
+ throw new RuntimeException("fail after applyDeletes");
}
}
if (!failed) {
@@ -855,7 +859,7 @@ public class TestIndexWriterDelete exten
System.out.println("TEST: now commit for failure");
}
modifier.commit();
- } catch (IOException ioe) {
+ } catch (RuntimeException ioe) {
// expected
failed = true;
}
@@ -906,6 +910,8 @@ public class TestIndexWriterDelete exten
String[] text = { "Amsterdam", "Venice" };
MockDirectoryWrapper dir = newMockDirectory();
+ // test uses IW unref'ed helper which is unaware of retries
+ dir.setEnableVirusScanner(false);
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
modifier.commit();
dir.failOn(failure.reset());
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Sep 19 13:24:09 2014
@@ -949,6 +949,7 @@ public class TestIndexWriterExceptions e
for (FailOnlyInCommit failure : failures) {
MockDirectoryWrapper dir = newMockDirectory();
dir.setFailOnCreateOutput(false);
+ dir.setEnableVirusScanner(false); // we check for specific list of files
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
Document doc = new Document();
doc.add(newTextField("field", "a field", Field.Store.YES));
@@ -1170,6 +1171,10 @@ public class TestIndexWriterExceptions e
public void testSimulatedCorruptIndex1() throws IOException {
BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we are corrupting it!
+ if (dir instanceof MockDirectoryWrapper) {
+ // we want to ensure our corruption always succeeds!
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter writer = null;
@@ -1218,6 +1223,10 @@ public class TestIndexWriterExceptions e
public void testSimulatedCorruptIndex2() throws IOException {
BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we are corrupting it!
+ if (dir instanceof MockDirectoryWrapper) {
+ // we want to ensure our corruption always succeeds!
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriter writer = null;
writer = new IndexWriter(
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Fri Sep 19 13:24:09 2014
@@ -61,6 +61,7 @@ public class TestIndexWriterOnDiskFull e
System.out.println("TEST: cycle: diskFree=" + diskFree);
}
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
+ dir.setEnableVirusScanner(false); // currently uses the IW unreferenced files method, unaware of retries
dir.setMaxSizeInBytes(diskFree);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
MergeScheduler ms = writer.getConfig().getMergeScheduler();
@@ -532,6 +533,7 @@ public class TestIndexWriterOnDiskFull e
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2)
.setMergeScheduler(new ConcurrentMergeScheduler()));
+ writer.commit(); // empty commit, to not create confusing situation with first commit
dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
final Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Fri Sep 19 13:24:09 2014
@@ -554,6 +554,8 @@ public class TestIndexWriterWithThreads
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
writerRef.set(new IndexWriter(d, newIndexWriterConfig(analyzer)));
+ // Make initial commit so the test doesn't trip "corrupt first commit" when virus checker refuses to delete partial segments_N file:
+ writerRef.get().commit();
final LineFileDocs docs = new LineFileDocs(random());
final Thread[] threads = new Thread[threadCount];
final int iters = atLeast(100);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java Fri Sep 19 13:24:09 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Sorted
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -1160,6 +1161,10 @@ public class TestNumericDocValuesUpdates
@Test
public void testDeleteUnusedUpdatesFiles() throws Exception {
Directory dir = newDirectory();
+ // test explicitly needs files to always be actually deleted
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+ }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java Fri Sep 19 13:24:09 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -189,6 +190,10 @@ public class TestOmitPositions extends L
// Verifies no *.prx exists when all fields omit term positions:
public void testNoPrxFile() throws Throwable {
Directory ram = newDirectory();
+ if (ram instanceof MockDirectoryWrapper) {
+ // we verify some files get deleted
+ ((MockDirectoryWrapper)ram).setEnableVirusScanner(false);
+ }
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
.setMaxBufferedDocs(3)
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java Fri Sep 19 13:24:09 2014
@@ -38,6 +38,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -219,6 +220,10 @@ public class TestOmitTf extends LuceneTe
// Verifies no *.prx exists when all fields omit term freq:
public void testNoPrxFile() throws Throwable {
Directory ram = newDirectory();
+ if (ram instanceof MockDirectoryWrapper) {
+ // we verify some files get deleted
+ ((MockDirectoryWrapper)ram).setEnableVirusScanner(false);
+ }
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
.setMaxBufferedDocs(3)
Modified: lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1626204&r1=1626203&r2=1626204&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene_solr_4_10/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java Fri Sep 19 13:24:09 2014
@@ -50,6 +50,7 @@ public class TestPersistentSnapshotDelet
public void testExistingSnapshots() throws Exception {
int numSnapshots = 3;
MockDirectoryWrapper dir = newMockDirectory();
+ dir.setEnableVirusScanner(false); // test relies on files actually being deleted
IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy(dir)));
PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
assertNull(psdp.getLastSaveFile());