You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/02/03 10:23:14 UTC
[1/5] lucene-solr git commit: migrate current patch from svn
Repository: lucene-solr
Updated Branches:
refs/heads/lucene-6835 [created] b4a2bf2b4
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
index bfec448..da03aea 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collection;
import java.util.Set;
import org.apache.lucene.index.IndexFileNames;
@@ -325,9 +326,11 @@ public class BlockDirectory extends FilterDirectory {
return dest;
}
- public void deleteFile(String name) throws IOException {
- cache.delete(getFileCacheName(name));
- super.deleteFile(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ for(String name : names) {
+ cache.delete(getFileCacheName(name));
+ }
+ super.deleteFiles(names);
}
public boolean isBlockCacheReadEnabled() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
index bee0f54..e5317bc 100644
--- a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
+++ b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java
@@ -144,10 +144,12 @@ public class HdfsDirectory extends BaseDirectory {
}
@Override
- public void deleteFile(String name) throws IOException {
- Path path = new Path(hdfsDirPath, name);
- LOG.debug("Deleting {}", path);
- getFileSystem().delete(path, false);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ for(String name : names) {
+ Path path = new Path(hdfsDirPath, name);
+ LOG.debug("Deleting {}", path);
+ getFileSystem().delete(path, false);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java b/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
index 153d324..e3a0e7b 100644
--- a/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
+++ b/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
@@ -18,6 +18,8 @@ package org.apache.solr.store.hdfs;
*/
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
@@ -88,9 +90,7 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
@Test
public void testWritingAndReadingAFile() throws IOException {
String[] listAll = directory.listAll();
- for (String file : listAll) {
- directory.deleteFile(file);
- }
+ directory.deleteFiles(Arrays.asList(listAll));
IndexOutput output = directory.createOutput("testing.test", new IOContext());
output.writeInt(12345);
@@ -117,15 +117,13 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
assertFalse(slowFileExists(directory, "testing.test.other"));
assertTrue(slowFileExists(directory, "testing.test"));
- directory.deleteFile("testing.test");
+ directory.deleteFiles(Collections.singleton("testing.test"));
assertFalse(slowFileExists(directory, "testing.test"));
}
public void testRename() throws IOException {
String[] listAll = directory.listAll();
- for (String file : listAll) {
- directory.deleteFile(file);
- }
+ directory.deleteFiles(Arrays.asList(listAll));
IndexOutput output = directory.createOutput("testing.test", new IOContext());
output.writeInt(12345);
@@ -137,7 +135,7 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
assertEquals(12345, input.readInt());
assertEquals(input.getFilePointer(), input.length());
input.close();
- directory.deleteFile("testing.test.renamed");
+ directory.deleteFiles(Collections.singleton("testing.test.renamed"));
assertFalse(slowFileExists(directory, "testing.test.renamed"));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java b/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
index 7f07f85..2f4803d 100644
--- a/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
+++ b/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
@@ -69,9 +69,6 @@ public class MockDirectoryFactory extends EphemeralDirectoryFactory {
// already been created.
mockDirWrapper.setPreventDoubleWrite(false);
- // IndexFetcher & co don't seem ready for this:
- mockDirWrapper.setEnableVirusScanner(false);
-
if (allowReadingFilesStillOpenForWrite) {
mockDirWrapper.setAllowReadingFilesStillOpenForWrite(true);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java b/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
index 1c3bbcd..81eaca0 100644
--- a/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
+++ b/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
@@ -51,7 +51,6 @@ public class MockFSDirectoryFactory extends StandardDirectoryFactory {
if (cdir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)cdir).setAssertNoUnrefencedFilesOnClose(false);
((MockDirectoryWrapper)cdir).setPreventDoubleWrite(false);
- ((MockDirectoryWrapper)cdir).setEnableVirusScanner(false);
}
return dir;
}
[4/5] lucene-solr git commit: migrate current patch from svn
Posted by mi...@apache.org.
migrate current patch from svn
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/84f44589
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/84f44589
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/84f44589
Branch: refs/heads/lucene-6835
Commit: 84f44589f6f91250666b5f66f7c7667d347e1e3a
Parents: a48ba50
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Feb 2 15:53:16 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Feb 2 15:53:16 2016 -0500
----------------------------------------------------------------------
.../lucene/analysis/hunspell/Dictionary.java | 4 +-
.../analysis/hunspell/Test64kAffixes.java | 4 +-
.../analysis/hunspell/TestAllDictionaries.java | 6 +-
.../analysis/hunspell/TestAllDictionaries2.java | 6 -
.../analysis/hunspell/TestDictionary.java | 6 +-
.../hunspell/TestHunspellStemFilter.java | 6 +-
.../index/TestBackwardsCompatibility.java | 6 -
.../simpletext/SimpleTextCompoundFormat.java | 2 +-
.../codecs/lucene50/Lucene50CompoundReader.java | 16 +-
.../apache/lucene/index/IndexFileDeleter.java | 128 +--
.../org/apache/lucene/index/IndexWriter.java | 12 +-
.../index/PersistentSnapshotDeletionPolicy.java | 2 +-
.../java/org/apache/lucene/store/Directory.java | 6 +-
.../org/apache/lucene/store/FSDirectory.java | 114 ++-
.../lucene/store/FileSwitchDirectory.java | 17 +-
.../apache/lucene/store/FilterDirectory.java | 4 +-
.../store/LockValidatingDirectoryWrapper.java | 4 +-
.../org/apache/lucene/store/MMapDirectory.java | 1 +
.../org/apache/lucene/store/NIOFSDirectory.java | 2 +-
.../lucene/store/NRTCachingDirectory.java | 27 +-
.../org/apache/lucene/store/RAMDirectory.java | 16 +-
.../apache/lucene/store/SimpleFSDirectory.java | 1 +
.../lucene/store/TrackingDirectoryWrapper.java | 9 +-
.../java/org/apache/lucene/util/IOUtils.java | 34 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 7 +-
.../lucene/util/bkd/OfflinePointWriter.java | 4 +-
.../org/apache/lucene/index/TestAddIndexes.java | 2 -
.../index/TestAllFilesCheckIndexHeader.java | 5 -
.../index/TestAllFilesDetectTruncation.java | 5 -
.../apache/lucene/index/TestAtomicUpdate.java | 1 -
.../index/TestBinaryDocValuesUpdates.java | 4 -
.../lucene/index/TestCodecHoldsOpenFiles.java | 11 +-
.../index/TestConcurrentMergeScheduler.java | 4 -
.../apache/lucene/index/TestDeletionPolicy.java | 44 +-
.../index/TestDemoParallelLeafReader.java | 13 +-
.../lucene/index/TestDirectoryReader.java | 54 +-
.../lucene/index/TestDirectoryReaderReopen.java | 15 +-
.../test/org/apache/lucene/index/TestDoc.java | 9 +-
.../apache/lucene/index/TestFieldsReader.java | 65 +-
.../lucene/index/TestIndexFileDeleter.java | 4 +-
.../apache/lucene/index/TestIndexWriter.java | 928 +++++++++----------
.../lucene/index/TestIndexWriterCommit.java | 20 -
.../lucene/index/TestIndexWriterDelete.java | 9 +-
.../lucene/index/TestIndexWriterExceptions.java | 17 +-
.../lucene/index/TestIndexWriterForceMerge.java | 1 -
.../lucene/index/TestIndexWriterFromReader.java | 17 +-
.../lucene/index/TestIndexWriterOnDiskFull.java | 1 -
.../lucene/index/TestNRTReaderCleanup.java | 14 +-
.../apache/lucene/index/TestNeverDelete.java | 8 -
.../index/TestNumericDocValuesUpdates.java | 4 -
.../apache/lucene/index/TestOmitPositions.java | 5 +-
.../org/apache/lucene/index/TestOmitTf.java | 5 +-
.../TestPersistentSnapshotDeletionPolicy.java | 2 +-
.../apache/lucene/index/TestRollingUpdates.java | 4 -
.../index/TestSnapshotDeletionPolicy.java | 22 +-
.../lucene/index/TestSwappedIndexFiles.java | 9 -
.../apache/lucene/search/TestPointQueries.java | 13 +-
.../lucene/store/TestBufferedIndexInput.java | 86 +-
.../org/apache/lucene/store/TestDirectory.java | 22 +-
.../lucene/store/TestFileSwitchDirectory.java | 1 -
.../lucene/store/TestNativeFSLockFactory.java | 23 +-
.../apache/lucene/store/TestRAMDirectory.java | 1 -
.../lucene/store/TestSimpleFSLockFactory.java | 5 +-
.../store/TestTrackingDirectoryWrapper.java | 2 +-
.../apache/lucene/util/TestOfflineSorter.java | 30 +-
.../org/apache/lucene/util/bkd/TestBKD.java | 11 +-
.../org/apache/lucene/util/fst/Test2BFST.java | 8 +-
.../org/apache/lucene/util/fst/TestFSTs.java | 5 +-
.../lucene/util/packed/TestPackedInts.java | 3 +-
.../apache/lucene/util/fst/TestFSTsMisc.java | 1 -
.../IndexAndTaxonomyReplicationClientTest.java | 17 +-
.../IndexAndTaxonomyRevisionTest.java | 8 -
.../lucene/replicator/IndexRevisionTest.java | 4 -
.../lucene/replicator/LocalReplicatorTest.java | 90 +-
.../lucene/util/BaseGeoPointTestCase.java | 9 -
.../org/apache/lucene/geo3d/TestGeo3DPoint.java | 11 +-
.../suggest/analyzing/AnalyzingSuggester.java | 2 +-
.../search/suggest/fst/ExternalRefSorter.java | 3 +-
.../search/suggest/fst/FSTCompletionLookup.java | 2 +-
.../lucene/search/suggest/PersistenceTest.java | 6 +-
.../search/suggest/TestInputIterator.java | 6 +-
.../analyzing/AnalyzingSuggesterTest.java | 6 +-
.../suggest/analyzing/FuzzySuggesterTest.java | 6 +-
.../search/suggest/fst/BytesRefSortersTest.java | 3 -
.../search/suggest/fst/FSTCompletionTest.java | 6 +-
.../search/suggest/fst/WFSTCompletionTest.java | 6 +-
.../index/BaseCompoundFormatTestCase.java | 7 +-
.../index/BaseIndexFileFormatTestCase.java | 11 +-
.../lucene/index/BasePointFormatTestCase.java | 11 -
.../index/BasePostingsFormatTestCase.java | 2 -
.../lucene/index/RandomPostingsTester.java | 1 -
.../ThreadedIndexingAndSearchingTestCase.java | 1 -
.../apache/lucene/mockfile/VirusCheckingFS.java | 65 ++
.../lucene/store/BaseDirectoryTestCase.java | 10 +-
.../lucene/store/BaseLockFactoryTestCase.java | 50 +-
.../lucene/store/MockDirectoryWrapper.java | 206 ++--
.../util/TestRuleTemporaryFilesCleanup.java | 19 +
.../java/org/apache/lucene/util/TestUtil.java | 46 +
.../org/apache/lucene/util/fst/FSTTester.java | 2 +-
.../TestCompressingStoredFieldsFormat.java | 4 -
.../lucene/mockfile/TestVirusCheckingFS.java | 62 ++
.../apache/lucene/mockfile/TestWindowsFS.java | 2 +-
.../lucene/store/TestMockDirectoryWrapper.java | 1 -
.../org/apache/solr/core/DirectoryFactory.java | 7 +-
.../org/apache/solr/handler/IndexFetcher.java | 4 +-
.../org/apache/solr/handler/RestoreCore.java | 3 +-
.../solr/store/blockcache/BlockDirectory.java | 9 +-
.../apache/solr/store/hdfs/HdfsDirectory.java | 10 +-
.../solr/store/hdfs/HdfsDirectoryTest.java | 14 +-
.../apache/solr/core/MockDirectoryFactory.java | 3 -
.../solr/core/MockFSDirectoryFactory.java | 1 -
111 files changed, 1245 insertions(+), 1448 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index d5db839..49c7045 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -877,7 +877,7 @@ public class Dictionary {
success = true;
} finally {
if (success) {
- tempDir.deleteFile(unsorted.getName());
+ tempDir.deleteFiles(Collections.singleton(unsorted.getName()));
} else {
IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName());
}
@@ -966,7 +966,7 @@ public class Dictionary {
success2 = true;
} finally {
if (success2) {
- tempDir.deleteFile(sorted);
+ tempDir.deleteFiles(Collections.singleton(sorted));
} else {
IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
index 6fa6ecf..20d4c47 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
@@ -28,6 +28,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
/** Tests that > 64k affixes actually works and doesnt overflow some internal int */
public class Test64kAffixes extends LuceneTestCase {
@@ -54,9 +55,6 @@ public class Test64kAffixes extends LuceneTestCase {
dictWriter.close();
try (InputStream affStream = Files.newInputStream(affix); InputStream dictStream = Files.newInputStream(dict); Directory tempDir2 = newDirectory()) {
- if (tempDir2 instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) tempDir2).setEnableVirusScanner(false);
- }
Dictionary dictionary = new Dictionary(tempDir2, "dictionary", affStream, dictStream);
Stemmer stemmer = new Stemmer(dictionary);
// drinks should still stem to drink
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
index 4a66430..5470ded 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
@@ -215,10 +215,6 @@ public class TestAllDictionaries extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
index efc5d66..1317cfc 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
@@ -189,9 +189,6 @@ public class TestAllDictionaries2 extends LuceneTestCase {
try (InputStream dictionary = Files.newInputStream(dicEntry);
InputStream affix = Files.newInputStream(affEntry);
Directory tempDir = newDirectory()) {
- if (tempDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
- }
Dictionary dic = new Dictionary(tempDir, "dictionary", affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
@@ -226,9 +223,6 @@ public class TestAllDictionaries2 extends LuceneTestCase {
try (InputStream dictionary = Files.newInputStream(dicEntry);
InputStream affix = Files.newInputStream(affEntry);
Directory tempDir = newDirectory()) {
- if (tempDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
- }
new Dictionary(tempDir, "dictionary", affix, dictionary);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index 39feb1d..1d162c0 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -260,10 +260,6 @@ public class TestDictionary extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
index 69acc93..5cda81a 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilter.java
@@ -129,10 +129,6 @@ public class TestHunspellStemFilter extends BaseTokenStreamTestCase {
}
private static Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index f61599d..87cdea6 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1238,12 +1238,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
System.out.println("testUpgradeOldSingleSegmentIndexWithAdditions: index=" +name);
}
Directory dir = newDirectory(oldIndexDirs.get(name));
- if (dir instanceof MockDirectoryWrapper) {
- // we need to ensure we delete old commits for this test,
- // otherwise IndexUpgrader gets angry
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
-
assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));
// create a bunch of dummy segments
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
index c994df7..8398f66 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
@@ -148,7 +148,7 @@ public class SimpleTextCompoundFormat extends CompoundFormat {
public void sync(Collection<String> names) { throw new UnsupportedOperationException(); }
@Override
- public void deleteFile(String name) { throw new UnsupportedOperationException(); }
+ public void deleteFiles(Collection<String> name) { throw new UnsupportedOperationException(); }
@Override
public void renameFile(String source, String dest) { throw new UnsupportedOperationException(); }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
index b1cfc5d..a6ccfd4 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
@@ -17,6 +17,13 @@ package org.apache.lucene.codecs.lucene50;
* limitations under the License.
*/
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
@@ -29,13 +36,6 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.IOUtils;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* Class for accessing a compound stream.
* This class implements a directory, but is limited to only read operations.
@@ -161,7 +161,7 @@ final class Lucene50CompoundReader extends Directory {
/** Not implemented
* @throws UnsupportedOperationException always: not supported by CFS */
@Override
- public void deleteFile(String name) {
+ public void deleteFiles(Collection<String> name) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index e2b29c9..569471e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -79,11 +79,6 @@ import org.apache.lucene.util.InfoStream;
final class IndexFileDeleter implements Closeable {
- /* Files that we tried to delete but failed (likely
- * because they are open and we are running on Windows),
- * so we will retry them again later: */
- private final Set<String> deletable = new HashSet<>();
-
/* Reference count for all files in the index.
* Counts how many existing commits reference a file.
**/
@@ -220,6 +215,7 @@ final class IndexFileDeleter implements Closeable {
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
+ Set<String> toDelete = new HashSet<>();
for(Map.Entry<String, RefCount> entry : refCounts.entrySet() ) {
RefCount rc = entry.getValue();
final String fileName = entry.getKey();
@@ -231,10 +227,12 @@ final class IndexFileDeleter implements Closeable {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "init: removing unreferenced file \"" + fileName + "\"");
}
- deleteFile(fileName);
+ toDelete.add(fileName);
}
}
+ deleteFiles(toDelete);
+
// Finally, give policy a chance to remove things on
// startup:
policy.onInit(commits);
@@ -425,7 +423,7 @@ final class IndexFileDeleter implements Closeable {
*/
void refresh() throws IOException {
assert locked();
- deletable.clear();
+ Set<String> toDelete = new HashSet<>();
String[] files = directory.listAll();
@@ -445,15 +443,15 @@ final class IndexFileDeleter implements Closeable {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "refresh: removing newly created unreferenced file \"" + fileName + "\"");
}
- deletable.add(fileName);
+ toDelete.add(fileName);
}
}
- deletePendingFiles();
+ deleteFiles(toDelete);
}
@Override
- public void close() {
+ public void close() throws IOException {
// DecRef old files from the last checkpoint, if any:
assert locked();
@@ -464,8 +462,6 @@ final class IndexFileDeleter implements Closeable {
lastFiles.clear();
}
}
-
- deletePendingFiles();
}
/**
@@ -489,39 +485,6 @@ final class IndexFileDeleter implements Closeable {
}
}
- public void deletePendingFiles() {
- assert locked();
-
- // Clone the set because it will change as we iterate:
- List<String> toDelete = new ArrayList<>(deletable);
-
- // First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
- // before we remove any files they reference. If any delete of segments_N fails, we leave all other files
- // undeleted so index is never in a corrupt state:
- for (String fileName : toDelete) {
- RefCount rc = refCounts.get(fileName);
- if (rc != null && rc.count > 0) {
- // LUCENE-5904: should never happen! This means we are about to pending-delete a referenced index file
- throw new IllegalStateException("file \"" + fileName + "\" is in pending delete set but has non-zero refCount=" + rc.count);
- } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
- if (deleteFile(fileName) == false) {
- if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "failed to remove commit point \"" + fileName + "\"; skipping deletion of all other pending files");
- }
- return;
- }
- }
- }
-
- // Only delete other files if we were able to remove the segments_N files; this way we never
- // leave a corrupt commit in the index even in the presense of virus checkers:
- for(String fileName : toDelete) {
- if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
- deleteFile(fileName);
- }
- }
- }
-
/**
* For definition of "check point" see IndexWriter comments:
* "Clarification: Check Points (and commits)".
@@ -610,12 +573,15 @@ final class IndexFileDeleter implements Closeable {
}
/** Decrefs all provided files, even on exception; throws first exception hit, if any. */
- void decRef(Collection<String> files) {
+ void decRef(Collection<String> files) throws IOException {
assert locked();
+ Set<String> toDelete = new HashSet<>();
Throwable firstThrowable = null;
for(final String file : files) {
try {
- decRef(file);
+ if (decRef(file)) {
+ toDelete.add(file);
+ }
} catch (Throwable t) {
if (firstThrowable == null) {
// Save first exception and throw it in the end, but be sure to finish decRef all files
@@ -625,7 +591,7 @@ final class IndexFileDeleter implements Closeable {
}
try {
- deletePendingFiles();
+ deleteFiles(toDelete);
} catch (Throwable t) {
if (firstThrowable == null) {
// Save first exception and throw it in the end, but be sure to finish decRef all files
@@ -634,27 +600,31 @@ final class IndexFileDeleter implements Closeable {
}
// NOTE: does nothing if firstThrowable is null
- IOUtils.reThrowUnchecked(firstThrowable);
+ IOUtils.reThrow(firstThrowable);
}
/** Decrefs all provided files, ignoring any exceptions hit; call this if
* you are already handling an exception. */
void decRefWhileHandlingException(Collection<String> files) {
assert locked();
+ Set<String> toDelete = new HashSet<>();
for(final String file : files) {
try {
- decRef(file);
+ if (decRef(file)) {
+ toDelete.add(file);
+ }
} catch (Throwable t) {
}
}
try {
- deletePendingFiles();
+ deleteFiles(toDelete);
} catch (Throwable t) {
}
}
- private void decRef(String fileName) {
+ /** Returns true if the file should now be deleted. */
+ private boolean decRef(String fileName) {
assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
@@ -662,14 +632,13 @@ final class IndexFileDeleter implements Closeable {
infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
}
}
- if (0 == rc.DecRef()) {
+ if (rc.DecRef() == 0) {
// This file is no longer referenced by any past
// commit points nor by the in-memory SegmentInfos:
- try {
- deletable.add(fileName);
- } finally {
- refCounts.remove(fileName);
- }
+ refCounts.remove(fileName);
+ return true;
+ } else {
+ return false;
}
}
@@ -692,8 +661,6 @@ final class IndexFileDeleter implements Closeable {
RefCount rc;
if (!refCounts.containsKey(fileName)) {
rc = new RefCount(fileName);
- // We should never incRef a file we are already wanting to delete:
- assert deletable.contains(fileName) == false: "file \"" + fileName + "\" cannot be incRef'd: it's already pending delete";
refCounts.put(fileName, rc);
} else {
rc = refCounts.get(fileName);
@@ -705,6 +672,7 @@ final class IndexFileDeleter implements Closeable {
* (have not yet been incref'd). */
void deleteNewFiles(Collection<String> files) throws IOException {
assert locked();
+ Set<String> toDelete = new HashSet<>();
for (final String fileName: files) {
// NOTE: it's very unusual yet possible for the
// refCount to be present and 0: it can happen if you
@@ -716,45 +684,31 @@ final class IndexFileDeleter implements Closeable {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "will delete new file \"" + fileName + "\"");
}
- deletable.add(fileName);
+ toDelete.add(fileName);
}
}
- deletePendingFiles();
+ deleteFiles(toDelete);
}
- /** Returns true if the delete succeeded. Otherwise, the fileName is
- * added to the deletable set so we will retry the delete later, and
- * we return false. */
- private boolean deleteFile(String fileName) {
+ private void deleteFiles(Collection<String> names) throws IOException {
assert locked();
ensureOpen();
+ if (names.isEmpty()) {
+ return;
+ }
try {
if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "delete \"" + fileName + "\"");
+ infoStream.message("IFD", "delete \"" + names + "\"");
}
- directory.deleteFile(fileName);
- deletable.remove(fileName);
- return true;
- } catch (IOException e) { // if delete fails
-
+ directory.deleteFiles(names);
+ } catch (NoSuchFileException | FileNotFoundException e) { // if delete fails
// IndexWriter should only ask us to delete files it knows it wrote, so if we hit this, something is wrong!
- // LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state:
- assert Constants.WINDOWS || e instanceof NoSuchFileException == false: "hit unexpected NoSuchFileException: file=" + fileName;
- assert Constants.WINDOWS || e instanceof FileNotFoundException == false: "hit unexpected FileNotFoundException: file=" + fileName;
-
- // Some operating systems (e.g. Windows) don't
- // permit a file to be deleted while it is opened
- // for read (e.g. by another process or thread). So
- // we assume that when a delete fails it is because
- // the file is open in another process, and queue
- // the file for subsequent deletion.
-
- if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "unable to remove file \"" + fileName + "\": " + e.toString() + "; Will re-try later.");
+ if (Constants.WINDOWS) {
+ // LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state:
+ } else {
+ throw e;
}
- deletable.add(fileName);
- return false;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index b05e15a..e62cafb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -51,6 +51,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
@@ -753,6 +754,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* IO error
*/
public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
+ if (d instanceof FSDirectory && ((FSDirectory) d).checkPendingDeletions()) {
+ throw new IllegalArgumentException("Directory still has pending deleted files");
+ }
+
conf.setIndexWriter(this); // prevent reuse by other instances
config = conf;
infoStream = config.getInfoStream();
@@ -3569,8 +3574,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
}
- deleter.deletePendingFiles();
-
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "after commitMerge: " + segString());
}
@@ -4616,14 +4619,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
*/
public synchronized void deleteUnusedFiles() throws IOException {
ensureOpen(false);
- deleter.deletePendingFiles();
deleter.revisitPolicy();
}
- private synchronized void deletePendingFiles() throws IOException {
- deleter.deletePendingFiles();
- }
-
/**
* NOTE: this method creates a compound file for all files returned by
* info.files(). While, generally, this may include separate norms and
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java b/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
index 0464852..2a286ff 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
@@ -212,7 +212,7 @@ public class PersistentSnapshotDeletionPolicy extends SnapshotDeletionPolicy {
private synchronized void clearPriorSnapshots() throws IOException {
for(String file : dir.listAll()) {
if (file.startsWith(SNAPSHOTS_PREFIX)) {
- dir.deleteFile(file);
+ dir.deleteFiles(Collections.singleton(file));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/Directory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index 8aa5fa1..b9e5ad4 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -49,8 +49,9 @@ public abstract class Directory implements Closeable {
*/
public abstract String[] listAll() throws IOException;
- /** Removes an existing file in the directory. */
- public abstract void deleteFile(String name) throws IOException;
+ /** Removes the specified files from the directory. If an exception is thrown, behavior is undefined
+ * (none, some or all of the files may have in fact been deleted). */
+ public abstract void deleteFiles(Collection<String> name) throws IOException;
/**
* Returns the length of a file in the directory. This method follows the
@@ -67,7 +68,6 @@ public abstract class Directory implements Closeable {
*/
public abstract long fileLength(String name) throws IOException;
-
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
public abstract IndexOutput createOutput(String name, IOContext context) throws IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index d02c126..0e1d4e9 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -17,19 +17,25 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.FileNotFoundException;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
@@ -120,6 +126,12 @@ public abstract class FSDirectory extends BaseDirectory {
protected final Path directory; // The underlying filesystem directory
+
+ /** Files we previously tried to delete, but hit exception (on Windows) last time we tried.
+ * These files are in "pending delete" state, where we refuse to openInput or createOutput
+ * them, nor include them in .listAll. */
+ protected final Set<String> pendingDeletes = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
+
/** Used to generate temp file names in {@link #createTempOutput}. */
private final AtomicLong nextTempFileCounter = new AtomicLong();
@@ -193,11 +205,18 @@ public abstract class FSDirectory extends BaseDirectory {
*
* @throws IOException if there was an I/O error during listing */
public static String[] listAll(Path dir) throws IOException {
+ return listAll(dir, Collections.emptySet());
+ }
+
+ private static String[] listAll(Path dir, Set<String> skipNames) throws IOException {
List<String> entries = new ArrayList<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
for (Path path : stream) {
- entries.add(path.getFileName().toString());
+ String name = path.getFileName().toString();
+ if (skipNames.contains(name) == false) {
+ entries.add(name);
+ }
}
}
@@ -207,7 +226,7 @@ public abstract class FSDirectory extends BaseDirectory {
@Override
public String[] listAll() throws IOException {
ensureOpen();
- return listAll(directory);
+ return listAll(directory, pendingDeletes);
}
/** Returns the length in bytes of a file in the directory. */
@@ -219,9 +238,10 @@ public abstract class FSDirectory extends BaseDirectory {
/** Removes an existing file in the directory. */
@Override
- public void deleteFile(String name) throws IOException {
+ public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
- Files.delete(directory.resolve(name));
+ pendingDeletes.addAll(names);
+ deletePendingFiles();
}
/** Creates an IndexOutput for the file with the given name. */
@@ -247,9 +267,20 @@ public abstract class FSDirectory extends BaseDirectory {
}
protected void ensureCanWrite(String name) throws IOException {
+ deletePendingFiles();
+ if (pendingDeletes.contains(name)) {
+ throw new IOException("file \"" + name + "\" is pending delete and cannot be overwritten");
+ }
Files.deleteIfExists(directory.resolve(name)); // delete existing, if any
}
+ protected void ensureCanRead(String name) throws IOException {
+ deletePendingFiles();
+ if (pendingDeletes.contains(name)) {
+ throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be overwritten");
+ }
+ }
+
@Override
public void sync(Collection<String> names) throws IOException {
ensureOpen();
@@ -270,8 +301,9 @@ public abstract class FSDirectory extends BaseDirectory {
/** Closes the store to future operations. */
@Override
- public synchronized void close() {
+ public synchronized void close() throws IOException {
isOpen = false;
+ deletePendingFiles();
}
/** @return the underlying filesystem directory */
@@ -286,6 +318,74 @@ public abstract class FSDirectory extends BaseDirectory {
return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
}
+ protected void fsync(String name) throws IOException {
+ deletePendingFiles();
+ IOUtils.fsync(directory.resolve(name), false);
+ }
+
+ /** Returns true if the file was successfully removed. */
+ private boolean deleteFile(String name) throws IOException {
+ try {
+ Files.delete(directory.resolve(name));
+ pendingDeletes.remove(name);
+ return true;
+ } catch (NoSuchFileException | FileNotFoundException e) {
+ // We were asked to delete a non-existent file:
+ pendingDeletes.remove(name);
+ throw e;
+ } catch (IOException ioe) {
+ // On windows, a file delete can fail because there's still an open
+ // file handle against it. We record this in pendingDeletes and
+ // try again later.
+
+ // TODO: this is hacky/lenient (we don't know which IOException this is), and
+ // it should only happen on filesystems that can do this, so really we should
+ // move this logic to WindowsDirectory or something
+
+ // TODO: can/should we do if (Constants.WINDOWS) here, else throw the exc?
+ // but what about a Linux box with a CIFS mount?
+ //System.out.println("FS.deleteFile failed (" + ioe + "): will retry later");
+ pendingDeletes.add(name);
+ return false;
+ }
+ }
+
+ /** Tries to delete any pending deleted files, and returns true if
+ * there are still files that could not be deleted. */
+ public boolean checkPendingDeletions() throws IOException {
+ deletePendingFiles();
+ return pendingDeletes.isEmpty() == false;
+ }
+
+ /** Try to delete any pending files that we had previously tried to delete but failed
+ * because we are on Windows and the files were still
+ * held open. */
+ public void deletePendingFiles() throws IOException {
+ // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?
+
+ // Clone the set because it will change as we iterate:
+ List<String> toDelete = new ArrayList<>(pendingDeletes);
+
+ // First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
+ // before we remove any files they reference. If any delete of segments_N fails, we leave all other files
+ // undeleted so index is never in a corrupt state:
+ for (String fileName : toDelete) {
+ if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+ if (deleteFile(fileName) == false) {
+ return;
+ }
+ }
+ }
+
+ // Only delete other files if we were able to remove the segments_N files; this way we never
+ // leave a corrupt commit in the index even in the presense of virus checkers:
+ for(String fileName : toDelete) {
+ if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
+ deleteFile(fileName);
+ }
+ }
+ }
+
final class FSIndexOutput extends OutputStreamIndexOutput {
/**
* The maximum chunk size is 8192 bytes, because file channel mallocs
@@ -312,8 +412,4 @@ public abstract class FSDirectory extends BaseDirectory {
}, CHUNK_SIZE);
}
}
-
- protected void fsync(String name) throws IOException {
- IOUtils.fsync(directory.resolve(name), false);
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
index 0b986f4..13bc217 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
@@ -140,8 +140,21 @@ public class FileSwitchDirectory extends Directory {
}
@Override
- public void deleteFile(String name) throws IOException {
- getDirectory(name).deleteFile(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ Set<String> primaryToDelete = new HashSet<>();
+ Set<String> secondaryToDelete = new HashSet<>();
+ for(String name : names) {
+ if (getDirectory(name) == primaryDir) {
+ primaryToDelete.add(name);
+ } else {
+ secondaryToDelete.add(name);
+ }
+ }
+ try {
+ primaryDir.deleteFiles(primaryToDelete);
+ } finally {
+ secondaryDir.deleteFiles(secondaryToDelete);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
index 7c550c1..9ee2928 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
@@ -58,8 +58,8 @@ public class FilterDirectory extends Directory {
}
@Override
- public void deleteFile(String name) throws IOException {
- in.deleteFile(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ in.deleteFiles(names);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
index 389c56d..3ed659a 100644
--- a/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
@@ -33,9 +33,9 @@ public final class LockValidatingDirectoryWrapper extends FilterDirectory {
}
@Override
- public void deleteFile(String name) throws IOException {
+ public void deleteFiles(Collection<String> names) throws IOException {
writeLock.ensureValid();
- in.deleteFile(name);
+ in.deleteFiles(names);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 0808eb6..b6b4033 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -234,6 +234,7 @@ public class MMapDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
+ ensureCanRead(name);
Path path = directory.resolve(name);
try (FileChannel c = FileChannel.open(path, StandardOpenOption.READ)) {
final String resourceDescription = "MMapIndexInput(path=\"" + path.toString() + "\")";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
index b739290..3234592 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
@@ -74,10 +74,10 @@ public class NIOFSDirectory extends FSDirectory {
this(path, FSLockFactory.getDefault());
}
- /** Creates an IndexInput for the file with the given name. */
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
+ ensureCanRead(name);
Path path = getDirectory().resolve(name);
FileChannel fc = FileChannel.open(path, StandardOpenOption.READ);
return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
index fd5e3d7..1b8404b 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
@@ -112,14 +112,23 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
}
@Override
- public synchronized void deleteFile(String name) throws IOException {
+ public synchronized void deleteFiles(Collection<String> names) throws IOException {
if (VERBOSE) {
- System.out.println("nrtdir.deleteFile name=" + name);
+ System.out.println("nrtdir.deleteFiles names=" + names);
}
- if (cache.fileNameExists(name)) {
- cache.deleteFile(name);
- } else {
- in.deleteFile(name);
+ Set<String> cacheToDelete = new HashSet<>();
+ Set<String> toDelete = new HashSet<>();
+ for(String name : names) {
+ if (cache.fileNameExists(name)) {
+ cacheToDelete.add(name);
+ } else {
+ toDelete.add(name);
+ }
+ }
+ try {
+ cache.deleteFiles(cacheToDelete);
+ } finally {
+ in.deleteFiles(toDelete);
}
}
@@ -146,14 +155,14 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
System.out.println(" to cache");
}
try {
- in.deleteFile(name);
+ in.deleteFiles(Collections.singleton(name));
} catch (IOException ioe) {
// This is fine: file may not exist
}
return cache.createOutput(name, context);
} else {
try {
- cache.deleteFile(name);
+ cache.deleteFiles(Collections.singleton(name));
} catch (IOException ioe) {
// This is fine: file may not exist
}
@@ -323,7 +332,7 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
synchronized(this) {
// Must sync here because other sync methods have
// if (cache.fileNameExists(name)) { ... } else { ... }:
- cache.deleteFile(fileName);
+ cache.deleteFiles(Collections.singleton(fileName));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
index d1dc0d0..7be4679 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
@@ -157,14 +157,16 @@ public class RAMDirectory extends BaseDirectory implements Accountable {
}
@Override
- public void deleteFile(String name) throws IOException {
+ public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
- RAMFile file = fileMap.remove(name);
- if (file != null) {
- file.directory = null;
- sizeInBytes.addAndGet(-file.sizeInBytes);
- } else {
- throw new FileNotFoundException(name);
+ for(String name : names) {
+ RAMFile file = fileMap.remove(name);
+ if (file != null) {
+ file.directory = null;
+ sizeInBytes.addAndGet(-file.sizeInBytes);
+ } else {
+ throw new FileNotFoundException(name);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
index 2daf98f..dc7a92c 100644
--- a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
@@ -72,6 +72,7 @@ public class SimpleFSDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
+ ensureCanRead(name);
Path path = directory.resolve(name);
SeekableByteChannel channel = Files.newByteChannel(path, StandardOpenOption.READ);
return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path + "\")", channel, context);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
index aa7214c..49d70f8 100644
--- a/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -33,9 +34,11 @@ public final class TrackingDirectoryWrapper extends FilterDirectory {
}
@Override
- public void deleteFile(String name) throws IOException {
- in.deleteFile(name);
- createdFileNames.remove(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ in.deleteFiles(names);
+ for(String name : names) {
+ createdFileNames.remove(name);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
index 510545f..3be3d7f 100644
--- a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
+++ b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
@@ -38,8 +38,10 @@ import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -191,12 +193,10 @@ public final class IOUtils {
* Note that the files should not be null.
*/
public static void deleteFilesIgnoringExceptions(Directory dir, Collection<String> files) {
- for (String name : files) {
- try {
- dir.deleteFile(name);
- } catch (Throwable ignored) {
- // ignore
- }
+ try {
+ dir.deleteFiles(files);
+ } catch (Throwable ignored) {
+ // ignore
}
}
@@ -212,24 +212,18 @@ public final class IOUtils {
* completes normally if there were no exceptions.
*
* @param dir Directory to delete files from
- * @param files file names to delete
+ * @param names file names to delete
*/
- public static void deleteFiles(Directory dir, Collection<String> files) throws IOException {
- Throwable th = null;
- for (String name : files) {
+ public static void deleteFiles(Directory dir, Collection<String> names) throws IOException {
+ Set<String> nonNullNames = new HashSet<>();
+ for(String name : names) {
if (name != null) {
- try {
- dir.deleteFile(name);
- } catch (Throwable t) {
- addSuppressed(th, t);
- if (th == null) {
- th = t;
- }
- }
+ nonNullNames.add(name);
}
}
-
- reThrow(th);
+ if (names.isEmpty() == false) {
+ dir.deleteFiles(names);
+ }
}
public static void deleteFiles(Directory dir, String... files) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index e47830d..328ea2e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -22,6 +22,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -39,8 +40,8 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntroSorter;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
+import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.StringHelper;
@@ -820,7 +821,7 @@ public class BKDWriter implements Closeable {
//System.out.println("sort time: " + ((t1-t0)/1000000.0) + " msec");
if (tempInput != null) {
- tempDir.deleteFile(tempInput.getName());
+ tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempInput = null;
} else {
assert heapPointWriter != null;
@@ -913,7 +914,7 @@ public class BKDWriter implements Closeable {
try {
tempInput.close();
} finally {
- tempDir.deleteFile(tempInput.getName());
+ tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
tempInput = null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
index 6cf1097..0751354 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointWriter.java
@@ -18,7 +18,7 @@ package org.apache.lucene.util.bkd;
*/
import java.io.IOException;
-
+import java.util.Collections;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -75,7 +75,7 @@ final class OfflinePointWriter implements PointWriter {
@Override
public void destroy() throws IOException {
- tempDir.deleteFile(out.getName());
+ tempDir.deleteFiles(Collections.singleton(out.getName()));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
index 778a7eb..4b32ec8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
@@ -1103,7 +1103,6 @@ public class TestAddIndexes extends LuceneTestCase {
}
}
-
// LUCENE-2790: tests that the non CFS files were deleted by addIndexes
public void testNonCFSLeftovers() throws Exception {
Directory[] dirs = new Directory[2];
@@ -1121,7 +1120,6 @@ public class TestAddIndexes extends LuceneTestCase {
DirectoryReader[] readers = new DirectoryReader[] { DirectoryReader.open(dirs[0]), DirectoryReader.open(dirs[1]) };
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
- dir.setEnableVirusScanner(false); // we check for specific list of files
IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(true));
MergePolicy lmp = conf.getMergePolicy();
// Force creation of CFS:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
index 3031553..cafe0a3 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
@@ -42,11 +42,6 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
-
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(TestUtil.getDefaultCodec());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
index 347955d..bf5625a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
@@ -42,11 +42,6 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
-
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(TestUtil.getDefaultCodec());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java b/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java
index e801bcc..acf1ebe 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java
@@ -181,6 +181,5 @@ public class TestAtomicUpdate extends LuceneTestCase {
directory = newFSDirectory(dirPath);
runTest(directory);
directory.close();
- IOUtils.rm(dirPath);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
index 9d72bd0..90d3c1c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
@@ -1094,10 +1094,6 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
public void testDeleteUnusedUpdatesFiles() throws Exception {
Directory dir = newDirectory();
- // test explicitly needs files to always be actually deleted
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
index 8077545..fa773fa 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.IntPoint;
@@ -47,15 +48,7 @@ public class TestCodecHoldsOpenFiles extends LuceneTestCase {
w.commit();
w.close();
- for(String fileName : d.listAll()) {
- try {
- d.deleteFile(fileName);
- // may succeed, e.g. if the file is completely read into RAM.
- } catch (IOException ioe) {
- // ignore: this means codec (correctly) is holding
- // the file open
- }
- }
+ d.deleteFiles(Arrays.asList(d.listAll()));
for(LeafReaderContext cxt : r.leaves()) {
TestUtil.checkReader(cxt.reader());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
index 083e426..a4b81de 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
@@ -184,10 +184,6 @@ public class TestConcurrentMergeScheduler extends LuceneTestCase {
public void testNoExtraFiles() throws IOException {
Directory directory = newDirectory();
- if (directory instanceof MockDirectoryWrapper) {
- // test uses IW unref'ed helper which is unaware of retries
- ((MockDirectoryWrapper)directory).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(2));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
index b56b1a0..6d7d1de 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
@@ -17,6 +17,15 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -27,23 +36,16 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/*
Verify we can read the pre-2.1 file format, do searches
against it, and add documents to it.
*/
-
+@SuppressFileSystems("VirusCheckingFS")
public class TestDeletionPolicy extends LuceneTestCase {
private void verifyCommitOrder(List<? extends IndexCommit> commits) {
@@ -223,10 +225,6 @@ public class TestDeletionPolicy extends LuceneTestCase {
final double SECONDS = 2.0;
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test manually deletes files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
.setIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(dir, SECONDS));
MergePolicy mp = conf.getMergePolicy();
@@ -299,7 +297,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
break;
}
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
gen--;
}
@@ -319,10 +317,6 @@ public class TestDeletionPolicy extends LuceneTestCase {
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test manually deletes files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
.setIndexDeletionPolicy(new KeepAllDeletionPolicy(dir))
@@ -381,7 +375,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
while(gen > 0) {
IndexReader reader = DirectoryReader.open(dir);
reader.close();
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
gen--;
if (gen > 0) {
@@ -570,10 +564,6 @@ public class TestDeletionPolicy extends LuceneTestCase {
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test manually deletes files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
for(int j=0;j<N+1;j++) {
@@ -612,7 +602,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
}
}
if (i < N) {
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
}
gen--;
}
@@ -634,10 +624,6 @@ public class TestDeletionPolicy extends LuceneTestCase {
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test manually deletes files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE)
.setIndexDeletionPolicy(new KeepLastNDeletionPolicy(N))
@@ -730,7 +716,7 @@ public class TestDeletionPolicy extends LuceneTestCase {
}
}
if (i < N) {
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
}
gen--;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
index 0d25f28..985606f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
@@ -384,7 +384,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
final Directory dir = openDirectory(leafIndex);
- if (Files.exists(leafIndex.resolve("done")) == false) {
+ if (slowFileExists(dir, "done") == false) {
if (DEBUG) System.out.println(Thread.currentThread().getName() + ": TEST: build segment index for " + leaf + " " + segIDGen + " (source: " + info.getDiagnostics().get("source") + ") dir=" + leafIndex);
if (dir.listAll().length != 0) {
@@ -895,7 +895,9 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
AtomicLong currentSchemaGen = new AtomicLong();
// TODO: separate refresh thread, search threads, indexing threads
- ReindexingReader reindexer = getReindexerNewDVFields(createTempDir(), currentSchemaGen);
+ Path root = createTempDir();
+ assumeFalse("we directly delete files", TestUtil.hasVirusChecker(root));
+ ReindexingReader reindexer = getReindexerNewDVFields(root, currentSchemaGen);
reindexer.commit();
Document doc = new Document();
@@ -965,6 +967,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
int numDocs = atLeast(TEST_NIGHTLY ? 20000 : 1000);
int maxID = 0;
Path root = createTempDir();
+ assumeFalse("we directly delete files", TestUtil.hasVirusChecker(root));
int refreshEveryNumDocs = 100;
int commitCloseNumDocs = 1000;
for(int i=0;i<numDocs;i++) {
@@ -1050,6 +1053,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
int numDocs = atLeast(TEST_NIGHTLY ? 20000 : 1000);
int maxID = 0;
Path root = createTempDir();
+ assumeFalse("we directly delete files", TestUtil.hasVirusChecker(root));
int refreshEveryNumDocs = 100;
int commitCloseNumDocs = 1000;
@@ -1151,7 +1155,9 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
}
public void testBasic() throws Exception {
- ReindexingReader reindexer = getReindexer(createTempDir());
+ Path tempPath = createTempDir();
+ assumeFalse("we directly delete files", TestUtil.hasVirusChecker(tempPath));
+ ReindexingReader reindexer = getReindexer(tempPath);
// Start with initial empty commit:
reindexer.commit();
@@ -1220,6 +1226,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
public void testRandom() throws Exception {
Path root = createTempDir();
+ assumeFalse("we directly delete files", TestUtil.hasVirusChecker(root));
ReindexingReader reindexer = null;
// TODO: separate refresh thread, search threads, indexing threads
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 00b424f..2c4f392 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -436,40 +436,43 @@ void assertTermDocsCount(String msg,
rmDir(fileDirName);
}*/
-public void testFilesOpenClose() throws IOException {
- // Create initial data set
- Path dirFile = createTempDir("TestIndexReader.testFilesOpenClose");
- Directory dir = newFSDirectory(dirFile);
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- addDoc(writer, "test");
- writer.close();
- dir.close();
+ public void testFilesOpenClose() throws IOException {
+ // Create initial data set
+ Path dirFile = createTempDir("TestIndexReader.testFilesOpenClose");
+ assumeFalse("test directly deletes files", TestUtil.hasVirusChecker(dirFile));
+ Directory dir = newFSDirectory(dirFile);
+
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ addDoc(writer, "test");
+ writer.close();
+ dir.close();
- // Try to erase the data - this ensures that the writer closed all files
- IOUtils.rm(dirFile);
- dir = newFSDirectory(dirFile);
+ // Try to erase the data - this ensures that the writer closed all files
+ IOUtils.rm(dirFile);
+ dir = newFSDirectory(dirFile);
- // Now create the data set again, just as before
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setOpenMode(OpenMode.CREATE));
- addDoc(writer, "test");
- writer.close();
- dir.close();
+ // Now create the data set again, just as before
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE));
+ addDoc(writer, "test");
+ writer.close();
+ dir.close();
- // Now open existing directory and test that reader closes all files
- dir = newFSDirectory(dirFile);
- DirectoryReader reader1 = DirectoryReader.open(dir);
- reader1.close();
- dir.close();
+ // Now open existing directory and test that reader closes all files
+ dir = newFSDirectory(dirFile);
+ DirectoryReader reader1 = DirectoryReader.open(dir);
+ reader1.close();
+ dir.close();
- // The following will fail if reader did not close
- // all files
- IOUtils.rm(dirFile);
+ // The following will fail if reader did not close
+ // all files
+ IOUtils.rm(dirFile);
}
public void testOpenReaderAfterDelete() throws IOException {
Path dirFile = createTempDir("deletetest");
Directory dir = newFSDirectory(dirFile);
+ assumeFalse("test deletes files directly", TestUtil.hasVirusChecker(dir));
if (dir instanceof BaseDirectoryWrapper) {
((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we will hit NoSuchFileException in MDW since we nuked it!
}
@@ -717,7 +720,6 @@ public void testFilesOpenClose() throws IOException {
// good exception
public void testNoDir() throws Throwable {
Path tempDir = createTempDir("doesnotexist");
- IOUtils.rm(tempDir);
Directory dir = newFSDirectory(tempDir);
try {
DirectoryReader.open(dir);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index 2c3b134..934ec73 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -39,14 +40,12 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-
-
public class TestDirectoryReaderReopen extends LuceneTestCase {
public void testReopen() throws Exception {
@@ -625,10 +624,6 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
public void testOverDecRefDuringReopen() throws Exception {
MockDirectoryWrapper dir = newMockDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // ensure we produce enough of our exceptions
- dir.setEnableVirusScanner(false);
- }
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
iwc.setCodec(TestUtil.getDefaultCodec());
@@ -713,7 +708,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
// Blow away the index:
for(String fileName : dir.listAll()) {
- dir.deleteFile(fileName);
+ dir.deleteFiles(Collections.singleton(fileName));
}
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
@@ -762,9 +757,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
DirectoryReader r = DirectoryReader.open(dir);
// Blow away the index:
- for(String fileName : dir.listAll()) {
- dir.deleteFile(fileName);
- }
+ dir.deleteFiles(Arrays.asList(dir.listAll()));
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
doc = new Document();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index 39fbec8..24ce405 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -48,6 +48,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
/** JUnit adaptation of an older test case DocTest. */
@@ -120,8 +121,6 @@ public class TestDoc extends LuceneTestCase {
// We create unreferenced files (we don't even write
// a segments file):
((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
- // this test itself deletes files (has no retry mechanism)
- ((MockDirectoryWrapper) directory).setEnableVirusScanner(false);
}
IndexWriter writer = new IndexWriter(
@@ -164,8 +163,6 @@ public class TestDoc extends LuceneTestCase {
// We create unreferenced files (we don't even write
// a segments file):
((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
- // this test itself deletes files (has no retry mechanism)
- ((MockDirectoryWrapper) directory).setEnableVirusScanner(false);
}
writer = new IndexWriter(
@@ -237,9 +234,7 @@ public class TestDoc extends LuceneTestCase {
Collection<String> filesToDelete = si.files();
codec.compoundFormat().write(dir, si, context);
si.setUseCompoundFile(true);
- for (final String fileToDelete : filesToDelete) {
- si1.info.dir.deleteFile(fileToDelete);
- }
+ si1.info.dir.deleteFiles(filesToDelete);
}
return new SegmentCommitInfo(si, 0, -1L, -1L, -1L);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
index d9a1d2e..2d33f71 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
@@ -188,42 +188,37 @@ public class TestFieldsReader extends LuceneTestCase {
public void testExceptions() throws Throwable {
Path indexDir = createTempDir("testfieldswriterexceptions");
- try {
- Directory fsDir = newFSDirectory(indexDir);
- FaultyFSDirectory dir = new FaultyFSDirectory(fsDir);
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()))
- .setOpenMode(OpenMode.CREATE);
- IndexWriter writer = new IndexWriter(dir, iwc);
- for(int i=0;i<2;i++)
- writer.addDocument(testDoc);
- writer.forceMerge(1);
- writer.close();
-
- IndexReader reader = DirectoryReader.open(dir);
- dir.startFailing();
-
- boolean exc = false;
-
- for(int i=0;i<2;i++) {
- try {
- reader.document(i);
- } catch (IOException ioe) {
- // expected
- exc = true;
- }
- try {
- reader.document(i);
- } catch (IOException ioe) {
- // expected
- exc = true;
- }
+ Directory fsDir = newFSDirectory(indexDir);
+ FaultyFSDirectory dir = new FaultyFSDirectory(fsDir);
+ IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ for(int i=0;i<2;i++)
+ writer.addDocument(testDoc);
+ writer.forceMerge(1);
+ writer.close();
+
+ IndexReader reader = DirectoryReader.open(dir);
+ dir.startFailing();
+
+ boolean exc = false;
+
+ for(int i=0;i<2;i++) {
+ try {
+ reader.document(i);
+ } catch (IOException ioe) {
+ // expected
+ exc = true;
+ }
+ try {
+ reader.document(i);
+ } catch (IOException ioe) {
+ // expected
+ exc = true;
}
- assertTrue(exc);
- reader.close();
- dir.close();
- } finally {
- IOUtils.rm(indexDir);
}
-
+ assertTrue(exc);
+ reader.close();
+ dir.close();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
index 5fa777b..fa9f1a7 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
@@ -34,6 +34,7 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
/*
Verify we can read the pre-2.1 file format, do searches
@@ -46,8 +47,6 @@ public class TestIndexFileDeleter extends LuceneTestCase {
Directory dir = newDirectory();
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
- // ensure we actually delete files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
}
MergePolicy mergePolicy = newLogMergePolicy(true, 10);
@@ -222,7 +221,6 @@ public class TestIndexFileDeleter extends LuceneTestCase {
public void testVirusScannerDoesntCorruptIndex() throws IOException {
MockDirectoryWrapper dir = newMockDirectory();
dir.setPreventDoubleWrite(false); // we arent trying to test this
- dir.setEnableVirusScanner(false); // we have our own to make test reproduce always
// add empty commit
new IndexWriter(dir, new IndexWriterConfig(null)).close();
[3/5] lucene-solr git commit: migrate current patch from svn
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index baa2484..3b45036 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -22,7 +22,11 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringReader;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -58,6 +62,8 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.mockfile.WindowsFS;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -67,10 +73,12 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.SimpleFSLockFactory;
@@ -90,494 +98,467 @@ import org.junit.Test;
public class TestIndexWriter extends LuceneTestCase {
- private static final FieldType storedTextType = new FieldType(TextField.TYPE_NOT_STORED);
- public void testDocCount() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = null;
- IndexReader reader = null;
- int i;
+ private static final FieldType storedTextType = new FieldType(TextField.TYPE_NOT_STORED);
+ public void testDocCount() throws IOException {
+ Directory dir = newDirectory();
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ IndexWriter writer = null;
+ IndexReader reader = null;
+ int i;
- // add 100 documents
- for (i = 0; i < 100; i++) {
- addDocWithIndex(writer,i);
- }
- assertEquals(100, writer.maxDoc());
- writer.close();
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- // delete 40 documents
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setMergePolicy(NoMergePolicy.INSTANCE));
- for (i = 0; i < 40; i++) {
- writer.deleteDocuments(new Term("id", ""+i));
- }
- writer.close();
+ // add 100 documents
+ for (i = 0; i < 100; i++) {
+ addDocWithIndex(writer,i);
+ }
+ assertEquals(100, writer.maxDoc());
+ writer.close();
- reader = DirectoryReader.open(dir);
- assertEquals(60, reader.numDocs());
- reader.close();
+ // delete 40 documents
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE));
+ for (i = 0; i < 40; i++) {
+ writer.deleteDocuments(new Term("id", ""+i));
+ }
+ writer.close();
- // merge the index down and check that the new doc count is correct
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- assertEquals(60, writer.numDocs());
- writer.forceMerge(1);
- assertEquals(60, writer.maxDoc());
- assertEquals(60, writer.numDocs());
- writer.close();
+ reader = DirectoryReader.open(dir);
+ assertEquals(60, reader.numDocs());
+ reader.close();
- // check that the index reader gives the same numbers.
- reader = DirectoryReader.open(dir);
- assertEquals(60, reader.maxDoc());
- assertEquals(60, reader.numDocs());
- reader.close();
-
- // make sure opening a new index for create over
- // this existing one works correctly:
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setOpenMode(OpenMode.CREATE));
- assertEquals(0, writer.maxDoc());
- assertEquals(0, writer.numDocs());
- writer.close();
- dir.close();
- }
+ // merge the index down and check that the new doc count is correct
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ assertEquals(60, writer.numDocs());
+ writer.forceMerge(1);
+ assertEquals(60, writer.maxDoc());
+ assertEquals(60, writer.numDocs());
+ writer.close();
- static void addDoc(IndexWriter writer) throws IOException
- {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa", Field.Store.NO));
- writer.addDocument(doc);
- }
+ // check that the index reader gives the same numbers.
+ reader = DirectoryReader.open(dir);
+ assertEquals(60, reader.maxDoc());
+ assertEquals(60, reader.numDocs());
+ reader.close();
- static void addDocWithIndex(IndexWriter writer, int index) throws IOException
- {
- Document doc = new Document();
- doc.add(newField("content", "aaa " + index, storedTextType));
- doc.add(newField("id", "" + index, storedTextType));
- writer.addDocument(doc);
- }
+ // make sure opening a new index for create over
+ // this existing one works correctly:
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE));
+ assertEquals(0, writer.maxDoc());
+ assertEquals(0, writer.numDocs());
+ writer.close();
+ dir.close();
+ }
+ static void addDoc(IndexWriter writer) throws IOException
+ {
+ Document doc = new Document();
+ doc.add(newTextField("content", "aaa", Field.Store.NO));
+ writer.addDocument(doc);
+ }
+ static void addDocWithIndex(IndexWriter writer, int index) throws IOException
+ {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa " + index, storedTextType));
+ doc.add(newField("id", "" + index, storedTextType));
+ writer.addDocument(doc);
+ }
- // TODO: we have the logic in MDW to do this check, and it's better, because it knows about files it tried
- // to delete but couldn't: we should replace this!!!!
- public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
- if (dir instanceof MockDirectoryWrapper) {
- assertFalse("test is broken: should disable virus scanner", ((MockDirectoryWrapper)dir).getEnableVirusScanner());
- }
- String[] startFiles = dir.listAll();
- new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))).rollback();
- String[] endFiles = dir.listAll();
+ // TODO: we have the logic in MDW to do this check, and it's better, because it knows about files it tried
+ // to delete but couldn't: we should replace this!!!!
+ public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
+ String[] startFiles = dir.listAll();
+ new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))).rollback();
+ String[] endFiles = dir.listAll();
- Arrays.sort(startFiles);
- Arrays.sort(endFiles);
+ Arrays.sort(startFiles);
+ Arrays.sort(endFiles);
- if (!Arrays.equals(startFiles, endFiles)) {
- fail(message + ": before delete:\n " + arrayToString(startFiles) + "\n after delete:\n " + arrayToString(endFiles));
- }
+ if (!Arrays.equals(startFiles, endFiles)) {
+ fail(message + ": before delete:\n " + arrayToString(startFiles) + "\n after delete:\n " + arrayToString(endFiles));
}
+ }
- static String arrayToString(String[] l) {
- String s = "";
- for(int i=0;i<l.length;i++) {
- if (i > 0) {
- s += "\n ";
- }
- s += l[i];
+ static String arrayToString(String[] l) {
+ String s = "";
+ for(int i=0;i<l.length;i++) {
+ if (i > 0) {
+ s += "\n ";
}
- return s;
+ s += l[i];
}
+ return s;
+ }
- // Make sure we can open an index for create even when a
- // reader holds it open (this fails pre lock-less
- // commits on windows):
- public void testCreateWithReader() throws IOException {
- Directory dir = newDirectory();
+ // Make sure we can open an index for create even when a
+ // reader holds it open (this fails pre lock-less
+ // commits on windows):
+ public void testCreateWithReader() throws IOException {
+ Directory dir = newDirectory();
- // add one document & close writer
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- addDoc(writer);
- writer.close();
+ // add one document & close writer
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ addDoc(writer);
+ writer.close();
- // now open reader:
- IndexReader reader = DirectoryReader.open(dir);
- assertEquals("should be one document", reader.numDocs(), 1);
+ // now open reader:
+ IndexReader reader = DirectoryReader.open(dir);
+ assertEquals("should be one document", reader.numDocs(), 1);
- // now open index for create:
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setOpenMode(OpenMode.CREATE));
- assertEquals("should be zero documents", writer.maxDoc(), 0);
- addDoc(writer);
- writer.close();
+ // now open index for create:
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE));
+ assertEquals("should be zero documents", writer.maxDoc(), 0);
+ addDoc(writer);
+ writer.close();
- assertEquals("should be one document", reader.numDocs(), 1);
- IndexReader reader2 = DirectoryReader.open(dir);
- assertEquals("should be one document", reader2.numDocs(), 1);
- reader.close();
- reader2.close();
+ assertEquals("should be one document", reader.numDocs(), 1);
+ IndexReader reader2 = DirectoryReader.open(dir);
+ assertEquals("should be one document", reader2.numDocs(), 1);
+ reader.close();
+ reader2.close();
- dir.close();
- }
+ dir.close();
+ }
- public void testChangesAfterClose() throws IOException {
- Directory dir = newDirectory();
+ public void testChangesAfterClose() throws IOException {
+ Directory dir = newDirectory();
- IndexWriter writer = null;
+ IndexWriter writer = null;
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- addDoc(writer);
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ addDoc(writer);
- // close
- writer.close();
- try {
- addDoc(writer);
- fail("did not hit AlreadyClosedException");
- } catch (AlreadyClosedException e) {
- // expected
- }
- dir.close();
+ // close
+ writer.close();
+ try {
+ addDoc(writer);
+ fail("did not hit AlreadyClosedException");
+ } catch (AlreadyClosedException e) {
+ // expected
}
+ dir.close();
+ }
- public void testIndexNoDocuments() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- writer.commit();
- writer.close();
+ public void testIndexNoDocuments() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ writer.commit();
+ writer.close();
- IndexReader reader = DirectoryReader.open(dir);
- assertEquals(0, reader.maxDoc());
- assertEquals(0, reader.numDocs());
- reader.close();
+ IndexReader reader = DirectoryReader.open(dir);
+ assertEquals(0, reader.maxDoc());
+ assertEquals(0, reader.numDocs());
+ reader.close();
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setOpenMode(OpenMode.APPEND));
- writer.commit();
- writer.close();
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.APPEND));
+ writer.commit();
+ writer.close();
- reader = DirectoryReader.open(dir);
- assertEquals(0, reader.maxDoc());
- assertEquals(0, reader.numDocs());
- reader.close();
- dir.close();
+ reader = DirectoryReader.open(dir);
+ assertEquals(0, reader.maxDoc());
+ assertEquals(0, reader.numDocs());
+ reader.close();
+ dir.close();
+ }
+
+ public void testSmallRAMBuffer() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setRAMBufferSizeMB(0.000001)
+ .setMergePolicy(newLogMergePolicy(10))
+ );
+ int lastNumSegments = getSegmentCount(dir);
+ for(int j=0;j<9;j++) {
+ Document doc = new Document();
+ doc.add(newField("field", "aaa" + j, storedTextType));
+ writer.addDocument(doc);
+ // Verify that with a tiny RAM buffer we see new
+ // segment after every doc
+ int numSegments = getSegmentCount(dir);
+ assertTrue(numSegments > lastNumSegments);
+ lastNumSegments = numSegments;
}
+ writer.close();
+ dir.close();
+ }
- public void testSmallRAMBuffer() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setRAMBufferSizeMB(0.000001)
- .setMergePolicy(newLogMergePolicy(10))
- );
- int lastNumSegments = getSegmentCount(dir);
- for(int j=0;j<9;j++) {
- Document doc = new Document();
- doc.add(newField("field", "aaa" + j, storedTextType));
- writer.addDocument(doc);
- // Verify that with a tiny RAM buffer we see new
- // segment after every doc
- int numSegments = getSegmentCount(dir);
- assertTrue(numSegments > lastNumSegments);
- lastNumSegments = numSegments;
- }
- writer.close();
- dir.close();
+ /** Returns how many unique segment names are in the directory. */
+ private static int getSegmentCount(Directory dir) throws IOException {
+ Set<String> segments = new HashSet<>();
+ for(String file : dir.listAll()) {
+ segments.add(IndexFileNames.parseSegmentName(file));
}
- /** Returns how many unique segment names are in the directory. */
- private static int getSegmentCount(Directory dir) throws IOException {
- Set<String> segments = new HashSet<>();
- for(String file : dir.listAll()) {
- segments.add(IndexFileNames.parseSegmentName(file));
- }
+ return segments.size();
+ }
- return segments.size();
- }
+ // Make sure it's OK to change RAM buffer size and
+ // maxBufferedDocs in a write session
+ public void testChangingRAMBuffer() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- // Make sure it's OK to change RAM buffer size and
- // maxBufferedDocs in a write session
- public void testChangingRAMBuffer() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- writer.getConfig().setMaxBufferedDocs(10);
- writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
-
- int lastFlushCount = -1;
- for(int j=1;j<52;j++) {
- Document doc = new Document();
- doc.add(new Field("field", "aaa" + j, storedTextType));
- writer.addDocument(doc);
- TestUtil.syncConcurrentMerges(writer);
- int flushCount = writer.getFlushCount();
- if (j == 1)
- lastFlushCount = flushCount;
- else if (j < 10)
- // No new files should be created
- assertEquals(flushCount, lastFlushCount);
- else if (10 == j) {
- assertTrue(flushCount > lastFlushCount);
- lastFlushCount = flushCount;
- writer.getConfig().setRAMBufferSizeMB(0.000001);
- writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- } else if (j < 20) {
- assertTrue(flushCount > lastFlushCount);
- lastFlushCount = flushCount;
- } else if (20 == j) {
- writer.getConfig().setRAMBufferSizeMB(16);
- writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- lastFlushCount = flushCount;
- } else if (j < 30) {
- assertEquals(flushCount, lastFlushCount);
- } else if (30 == j) {
- writer.getConfig().setRAMBufferSizeMB(0.000001);
- writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- } else if (j < 40) {
- assertTrue(flushCount> lastFlushCount);
- lastFlushCount = flushCount;
- } else if (40 == j) {
- writer.getConfig().setMaxBufferedDocs(10);
- writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- lastFlushCount = flushCount;
- } else if (j < 50) {
- assertEquals(flushCount, lastFlushCount);
- writer.getConfig().setMaxBufferedDocs(10);
- writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- } else if (50 == j) {
- assertTrue(flushCount > lastFlushCount);
- }
+ int lastFlushCount = -1;
+ for(int j=1;j<52;j++) {
+ Document doc = new Document();
+ doc.add(new Field("field", "aaa" + j, storedTextType));
+ writer.addDocument(doc);
+ TestUtil.syncConcurrentMerges(writer);
+ int flushCount = writer.getFlushCount();
+ if (j == 1)
+ lastFlushCount = flushCount;
+ else if (j < 10)
+ // No new files should be created
+ assertEquals(flushCount, lastFlushCount);
+ else if (10 == j) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (j < 20) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (20 == j) {
+ writer.getConfig().setRAMBufferSizeMB(16);
+ writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 30) {
+ assertEquals(flushCount, lastFlushCount);
+ } else if (30 == j) {
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (j < 40) {
+ assertTrue(flushCount> lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (40 == j) {
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 50) {
+ assertEquals(flushCount, lastFlushCount);
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (50 == j) {
+ assertTrue(flushCount > lastFlushCount);
}
- writer.close();
- dir.close();
}
+ writer.close();
+ dir.close();
+ }
- public void testChangingRAMBuffer2() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- writer.getConfig().setMaxBufferedDocs(10);
- writer.getConfig().setMaxBufferedDeleteTerms(10);
- writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ public void testChangingRAMBuffer2() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ writer.getConfig().setMaxBufferedDocs(10);
+ writer.getConfig().setMaxBufferedDeleteTerms(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- for(int j=1;j<52;j++) {
- Document doc = new Document();
- doc.add(new Field("field", "aaa" + j, storedTextType));
- writer.addDocument(doc);
- }
+ for(int j=1;j<52;j++) {
+ Document doc = new Document();
+ doc.add(new Field("field", "aaa" + j, storedTextType));
+ writer.addDocument(doc);
+ }
- int lastFlushCount = -1;
- for(int j=1;j<52;j++) {
- writer.deleteDocuments(new Term("field", "aaa" + j));
- TestUtil.syncConcurrentMerges(writer);
- int flushCount = writer.getFlushCount();
+ int lastFlushCount = -1;
+ for(int j=1;j<52;j++) {
+ writer.deleteDocuments(new Term("field", "aaa" + j));
+ TestUtil.syncConcurrentMerges(writer);
+ int flushCount = writer.getFlushCount();
- if (j == 1)
- lastFlushCount = flushCount;
- else if (j < 10) {
- // No new files should be created
- assertEquals(flushCount, lastFlushCount);
- } else if (10 == j) {
- assertTrue("" + j, flushCount > lastFlushCount);
- lastFlushCount = flushCount;
- writer.getConfig().setRAMBufferSizeMB(0.000001);
- writer.getConfig().setMaxBufferedDeleteTerms(1);
- } else if (j < 20) {
- assertTrue(flushCount > lastFlushCount);
- lastFlushCount = flushCount;
- } else if (20 == j) {
- writer.getConfig().setRAMBufferSizeMB(16);
- writer.getConfig().setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- lastFlushCount = flushCount;
- } else if (j < 30) {
- assertEquals(flushCount, lastFlushCount);
- } else if (30 == j) {
- writer.getConfig().setRAMBufferSizeMB(0.000001);
- writer.getConfig().setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- writer.getConfig().setMaxBufferedDeleteTerms(1);
- } else if (j < 40) {
- assertTrue(flushCount> lastFlushCount);
- lastFlushCount = flushCount;
- } else if (40 == j) {
- writer.getConfig().setMaxBufferedDeleteTerms(10);
- writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- lastFlushCount = flushCount;
- } else if (j < 50) {
- assertEquals(flushCount, lastFlushCount);
- writer.getConfig().setMaxBufferedDeleteTerms(10);
- writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
- } else if (50 == j) {
- assertTrue(flushCount > lastFlushCount);
- }
+ if (j == 1)
+ lastFlushCount = flushCount;
+ else if (j < 10) {
+ // No new files should be created
+ assertEquals(flushCount, lastFlushCount);
+ } else if (10 == j) {
+ assertTrue("" + j, flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDeleteTerms(1);
+ } else if (j < 20) {
+ assertTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (20 == j) {
+ writer.getConfig().setRAMBufferSizeMB(16);
+ writer.getConfig().setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 30) {
+ assertEquals(flushCount, lastFlushCount);
+ } else if (30 == j) {
+ writer.getConfig().setRAMBufferSizeMB(0.000001);
+ writer.getConfig().setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ writer.getConfig().setMaxBufferedDeleteTerms(1);
+ } else if (j < 40) {
+ assertTrue(flushCount> lastFlushCount);
+ lastFlushCount = flushCount;
+ } else if (40 == j) {
+ writer.getConfig().setMaxBufferedDeleteTerms(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ } else if (j < 50) {
+ assertEquals(flushCount, lastFlushCount);
+ writer.getConfig().setMaxBufferedDeleteTerms(10);
+ writer.getConfig().setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ } else if (50 == j) {
+ assertTrue(flushCount > lastFlushCount);
}
- writer.close();
- dir.close();
}
+ writer.close();
+ dir.close();
+ }
- public void testEnablingNorms() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setMaxBufferedDocs(10));
- // Enable norms for only 1 doc, pre flush
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setOmitNorms(true);
- for(int j=0;j<10;j++) {
- Document doc = new Document();
- Field f = null;
- if (j != 8) {
- f = newField("field", "aaa", customType);
- }
- else {
- f = newField("field", "aaa", storedTextType);
- }
- doc.add(f);
- writer.addDocument(doc);
+ public void testEnablingNorms() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxBufferedDocs(10));
+ // Enable norms for only 1 doc, pre flush
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setOmitNorms(true);
+ for(int j=0;j<10;j++) {
+ Document doc = new Document();
+ Field f = null;
+ if (j != 8) {
+ f = newField("field", "aaa", customType);
}
- writer.close();
-
- Term searchTerm = new Term("field", "aaa");
-
- IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = newSearcher(reader);
- ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), 1000).scoreDocs;
- assertEquals(10, hits.length);
- reader.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(10));
- // Enable norms for only 1 doc, post flush
- for(int j=0;j<27;j++) {
- Document doc = new Document();
- Field f = null;
- if (j != 26) {
- f = newField("field", "aaa", customType);
- }
- else {
- f = newField("field", "aaa", storedTextType);
- }
- doc.add(f);
- writer.addDocument(doc);
+ else {
+ f = newField("field", "aaa", storedTextType);
}
- writer.close();
- reader = DirectoryReader.open(dir);
- searcher = newSearcher(reader);
- hits = searcher.search(new TermQuery(searchTerm), 1000).scoreDocs;
- assertEquals(27, hits.length);
- reader.close();
-
- reader = DirectoryReader.open(dir);
- reader.close();
-
- dir.close();
+ doc.add(f);
+ writer.addDocument(doc);
}
+ writer.close();
- public void testHighFreqTerm() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
- .setRAMBufferSizeMB(0.01));
- // Massive doc that has 128 K a's
- StringBuilder b = new StringBuilder(1024*1024);
- for(int i=0;i<4096;i++) {
- b.append(" a a a a a a a a");
- b.append(" a a a a a a a a");
- b.append(" a a a a a a a a");
- b.append(" a a a a a a a a");
- }
- Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
- doc.add(newField("field", b.toString(), customType));
- writer.addDocument(doc);
- writer.close();
+ Term searchTerm = new Term("field", "aaa");
- IndexReader reader = DirectoryReader.open(dir);
- assertEquals(1, reader.maxDoc());
- assertEquals(1, reader.numDocs());
- Term t = new Term("field", "a");
- assertEquals(1, reader.docFreq(t));
- PostingsEnum td = TestUtil.docs(random(), reader,
- "field",
- new BytesRef("a"),
- null,
- PostingsEnum.FREQS);
- td.nextDoc();
- assertEquals(128*1024, td.freq());
- reader.close();
- dir.close();
- }
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = newSearcher(reader);
+ ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), 1000).scoreDocs;
+ assertEquals(10, hits.length);
+ reader.close();
- public void testFlushWithNoMerging() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(new MockAnalyzer(random()))
- .setMaxBufferedDocs(2)
- .setMergePolicy(newLogMergePolicy(10))
- );
+ writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(10));
+ // Enable norms for only 1 doc, post flush
+ for(int j=0;j<27;j++) {
Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
- doc.add(newField("field", "aaa", customType));
- for(int i=0;i<19;i++) {
- writer.addDocument(doc);
+ Field f = null;
+ if (j != 26) {
+ f = newField("field", "aaa", customType);
}
- writer.flush(false, true);
- writer.close();
- SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
- // Since we flushed w/o allowing merging we should now
- // have 10 segments
- assertEquals(10, sis.size());
- dir.close();
- }
-
- public void testFlushWithNoCommit() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- IndexWriter writer = new IndexWriter(dir, iwc);
- Document doc = new Document();
- writer.addDocument(doc);
- writer.commit();
-
+ else {
+ f = newField("field", "aaa", storedTextType);
+ }
+ doc.add(f);
writer.addDocument(doc);
- writer.flush();
- DirectoryReader r = DirectoryReader.open(dir);
- assertEquals(1, r.maxDoc());
- writer.commit();
- DirectoryReader r2 = DirectoryReader.openIfChanged(r);
- assertNotNull(r2);
- assertEquals(2, r2.maxDoc());
- IOUtils.close(r2, r, writer, dir);
}
+ writer.close();
+ reader = DirectoryReader.open(dir);
+ searcher = newSearcher(reader);
+ hits = searcher.search(new TermQuery(searchTerm), 1000).scoreDocs;
+ assertEquals(27, hits.length);
+ reader.close();
- // Make sure we can flush segment w/ norms, then add
- // empty doc (no norms) and flush
- public void testEmptyDocAfterFlushingRealDoc() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
- Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setStoreTermVectors(true);
- customType.setStoreTermVectorPositions(true);
- customType.setStoreTermVectorOffsets(true);
- doc.add(newField("field", "aaa", customType));
- writer.addDocument(doc);
- writer.commit();
- if (VERBOSE) {
- System.out.println("\nTEST: now add empty doc");
- }
- writer.addDocument(new Document());
- writer.close();
- IndexReader reader = DirectoryReader.open(dir);
- assertEquals(2, reader.numDocs());
- reader.close();
- dir.close();
+ reader = DirectoryReader.open(dir);
+ reader.close();
+
+ dir.close();
+ }
+
+ public void testHighFreqTerm() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
+ .setRAMBufferSizeMB(0.01));
+ // Massive doc that has 128 K a's
+ StringBuilder b = new StringBuilder(1024*1024);
+ for(int i=0;i<4096;i++) {
+ b.append(" a a a a a a a a");
+ b.append(" a a a a a a a a");
+ b.append(" a a a a a a a a");
+ b.append(" a a a a a a a a");
}
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectorOffsets(true);
+ doc.add(newField("field", b.toString(), customType));
+ writer.addDocument(doc);
+ writer.close();
+ IndexReader reader = DirectoryReader.open(dir);
+ assertEquals(1, reader.maxDoc());
+ assertEquals(1, reader.numDocs());
+ Term t = new Term("field", "a");
+ assertEquals(1, reader.docFreq(t));
+ PostingsEnum td = TestUtil.docs(random(), reader,
+ "field",
+ new BytesRef("a"),
+ null,
+ PostingsEnum.FREQS);
+ td.nextDoc();
+ assertEquals(128*1024, td.freq());
+ reader.close();
+ dir.close();
+ }
+
+ public void testFlushWithNoMerging() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxBufferedDocs(2)
+ .setMergePolicy(newLogMergePolicy(10))
+ );
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectorOffsets(true);
+ doc.add(newField("field", "aaa", customType));
+ for(int i=0;i<19;i++)
+ writer.addDocument(doc);
+ writer.flush(false, true);
+ writer.close();
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+ // Since we flushed w/o allowing merging we should now
+ // have 10 segments
+ assertEquals(10, sis.size());
+ dir.close();
+ }
+ // Make sure we can flush segment w/ norms, then add
+ // empty doc (no norms) and flush
+ public void testEmptyDocAfterFlushingRealDoc() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorPositions(true);
+ customType.setStoreTermVectorOffsets(true);
+ doc.add(newField("field", "aaa", customType));
+ writer.addDocument(doc);
+ writer.commit();
+ if (VERBOSE) {
+ System.out.println("\nTEST: now add empty doc");
+ }
+ writer.addDocument(new Document());
+ writer.close();
+ IndexReader reader = DirectoryReader.open(dir);
+ assertEquals(2, reader.numDocs());
+ reader.close();
+ dir.close();
+ }
/**
* Test that no NullPointerException will be raised,
@@ -1276,10 +1257,17 @@ public class TestIndexWriter extends LuceneTestCase {
public void testDeleteUnusedFiles() throws Exception {
+
assumeFalse("test relies on exact filenames", Codec.getDefault() instanceof SimpleTextCodec);
for(int iter=0;iter<2;iter++) {
- MockDirectoryWrapper dir = newMockDirectory(); // relies on windows semantics
- dir.setEnableVirusScanner(false); // but ensures files are actually deleted
+ // relies on windows semantics
+ Path path = createTempDir();
+ assumeFalse("test directly deletes files", TestUtil.hasVirusChecker(path));
+ FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
+ Path indexPath = new FilterPath(path, fs);
+
+ // NOTE: cannot use MMapDir, because WindowsFS doesn't see/think it keeps file handles open?
+ FSDirectory dir = new NIOFSDirectory(indexPath);
MergePolicy mergePolicy = newLogMergePolicy(true);
@@ -1306,20 +1294,15 @@ public class TestIndexWriter extends LuceneTestCase {
r = DirectoryReader.open(dir);
}
- List<String> files = new ArrayList<>(Arrays.asList(dir.listAll()));
-
- // RAMDir won't have a write.lock, but fs dirs will:
- files.remove("write.lock");
-
- assertTrue(files.contains("_0.cfs"));
- assertTrue(files.contains("_0.cfe"));
- assertTrue(files.contains("_0.si"));
+ assertTrue(Files.exists(indexPath.resolve("_0.cfs")));
+ assertTrue(Files.exists(indexPath.resolve("_0.cfe")));
+ assertTrue(Files.exists(indexPath.resolve("_0.si")));
if (iter == 1) {
// we run a full commit so there should be a segments file etc.
- assertTrue(files.contains("segments_1"));
+ assertTrue(Files.exists(indexPath.resolve("segments_1")));
} else {
// this is an NRT reopen - no segments files yet
- assertFalse(files.contains("segments_1"));
+ assertFalse(Files.exists(indexPath.resolve("segments_1")));
}
w.addDocument(doc);
w.forceMerge(1);
@@ -1329,34 +1312,29 @@ public class TestIndexWriter extends LuceneTestCase {
IndexReader r2 = DirectoryReader.openIfChanged(r);
assertNotNull(r2);
assertTrue(r != r2);
- files = Arrays.asList(dir.listAll());
// NOTE: here we rely on "Windows" behavior, ie, even
// though IW wanted to delete _0.cfs since it was
// merged away, because we have a reader open
// against this file, it should still be here:
- assertTrue(files.contains("_0.cfs"));
+ assertTrue(Files.exists(indexPath.resolve("_0.cfs")));
// forceMerge created this
//assertTrue(files.contains("_2.cfs"));
w.deleteUnusedFiles();
- files = Arrays.asList(dir.listAll());
// r still holds this file open
- assertTrue(files.contains("_0.cfs"));
+ assertTrue(Files.exists(indexPath.resolve("_0.cfs")));
//assertTrue(files.contains("_2.cfs"));
r.close();
if (iter == 0) {
// on closing NRT reader, it calls writer.deleteUnusedFiles
- files = Arrays.asList(dir.listAll());
- assertFalse(files.contains("_0.cfs"));
+ assertFalse(Files.exists(indexPath.resolve("_0.cfs")));
} else {
- // now writer can remove it
- w.deleteUnusedFiles();
- files = Arrays.asList(dir.listAll());
- assertFalse(files.contains("_0.cfs"));
+ // now FSDir can remove it
+ dir.deletePendingFiles();
+ assertFalse(Files.exists(indexPath.resolve("_0.cfs")));
}
- //assertTrue(files.contains("_2.cfs"));
w.close();
r2.close();
@@ -1369,10 +1347,6 @@ public class TestIndexWriter extends LuceneTestCase {
// Validates that iw.deleteUnusedFiles() also deletes unused index commits
// in case a deletion policy which holds onto commits is used.
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // otherwise the delete of old commit might not actually succeed temporarily.
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
@@ -1425,9 +1399,6 @@ public class TestIndexWriter extends LuceneTestCase {
// indexed, flushed (but not committed) and then IW rolls back, then no
// files are left in the Directory.
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
String[] origFiles = dir.listAll();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
@@ -1492,10 +1463,6 @@ public class TestIndexWriter extends LuceneTestCase {
public void testNoUnwantedTVFiles() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test uses IW unref'ed check which is unaware of retries
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setRAMBufferSizeMB(0.01)
.setMergePolicy(newLogMergePolicy()));
@@ -1661,7 +1628,6 @@ public class TestIndexWriter extends LuceneTestCase {
public void testDeleteAllNRTLeftoverFiles() throws Exception {
MockDirectoryWrapper d = new MockDirectoryWrapper(random(), new RAMDirectory());
- d.setEnableVirusScanner(false); // needs for files to actually be deleted
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
Document doc = new Document();
for(int i = 0; i < 20; i++) {
@@ -1784,10 +1750,6 @@ public class TestIndexWriter extends LuceneTestCase {
// LUCENE-3872
public void testPrepareCommitThenRollback() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // indexExists might return true if virus scanner prevents deletions
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter w = new IndexWriter(dir,
new IndexWriterConfig(new MockAnalyzer(random())));
@@ -2273,18 +2235,6 @@ public class TestIndexWriter extends LuceneTestCase {
dir.setCheckIndexOnClose(false);
}
- if (dir instanceof MockDirectoryWrapper) {
- MockDirectoryWrapper mdw = (MockDirectoryWrapper) dir;
- String[] files = dir.listAll();
- Arrays.sort(files);
- if ((Arrays.equals(new String[] {"segments_0"}, files) ||
- Arrays.equals(new String[] {"segments_0", "write.lock"}, files)) &&
- mdw.didTryToDelete("segments_0")) {
- // This means virus checker blocked IW deleting the corrupt first commit
- dir.setCheckIndexOnClose(false);
- }
- }
-
dir.close();
}
}
@@ -2733,7 +2683,7 @@ public class TestIndexWriter extends LuceneTestCase {
}
// LUCENE-6523
- public void testCommitImmediaatelyAfterNRTReopen() throws Exception {
+ public void testCommitImmediatelyAfterNRTReopen() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriter w = new IndexWriter(dir, iwc);
@@ -2753,11 +2703,29 @@ public class TestIndexWriter extends LuceneTestCase {
IOUtils.close(r, r2, w, dir);
}
+ // nocommit turn test on once we have VirusCheckingFS
+ /*
+ public void testWithPendingDeletions() throws Exception {
+ try (FSDirectory dir = FSDirectory.open(createTempDir())) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ IndexWriter w = new IndexWriter(dir, iwc);
+ w.commit();
+ IndexInput in = dir.openInput("segments_0", IOContext.DEFAULT);
+ w.addDocument(new Document());
+ w.close();
+ assertTrue(dir.checkPendingDeletions());
+ iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ try {
+ w = new IndexWriter(dir, iwc);
+ } catch (IllegalArgumentException iae) {
+ assertEquals("Directory still has pending deleted files", iae.getMessage());
+ }
+ }
+ }
+ */
+
public void testLeftoverTempFiles() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
IndexWriter w = new IndexWriter(dir, iwc);
w.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
index 3804717..4ac3f44 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
@@ -94,10 +94,6 @@ public class TestIndexWriterCommit extends LuceneTestCase {
*/
public void testCommitOnCloseAbort() throws IOException {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test uses IW unref'ed check which is unaware of retries
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(10));
for (int i = 0; i < 14; i++) {
@@ -189,11 +185,6 @@ public class TestIndexWriterCommit extends LuceneTestCase {
final String contentFormat = TestUtil.getPostingsFormat("content");
assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory"));
MockDirectoryWrapper dir = newMockDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // the virus scanner can use up too much disk space :)
- // an alternative is to expose MDW.triedToDelete and discount it
- dir.setEnableVirusScanner(false);
- }
Analyzer analyzer;
if (random().nextBoolean()) {
// no payloads
@@ -279,10 +270,6 @@ public class TestIndexWriterCommit extends LuceneTestCase {
if (dir instanceof MockDirectoryWrapper) {
((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
}
- if (dir instanceof MockDirectoryWrapper) {
- // test uses IW unref'ed check which is unaware of retries
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(
dir,
newIndexWriterConfig(new MockAnalyzer(random()))
@@ -599,14 +586,7 @@ public class TestIndexWriterCommit extends LuceneTestCase {
IndexReader reader2 = DirectoryReader.open(dir);
assertEquals(0, reader2.numDocs());
- // We need to let IW delete the partial segments_N that was written in prepareCommit, else we get a false fail below:
- if (mockDir != null) {
- mockDir.setEnableVirusScanner(false);
- }
writer.rollback();
- if (mockDir != null) {
- mockDir.setEnableVirusScanner(true);
- }
IndexReader reader3 = DirectoryReader.openIfChanged(reader);
assertNull(reader3);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
index 9213c25..be98ef0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
@@ -498,10 +498,7 @@ public class TestIndexWriterDelete extends LuceneTestCase {
// First build up a starting index:
MockDirectoryWrapper startDir = newMockDirectory();
- // TODO: find the resource leak that only occurs sometimes here.
- startDir.setNoDeleteOpenFile(false);
- // test uses IW unref'ed helper which is unaware of retries
- startDir.setEnableVirusScanner(false);
+
IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
for (int i = 0; i < 157; i++) {
Document d = new Document();
@@ -527,8 +524,6 @@ public class TestIndexWriterDelete extends LuceneTestCase {
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(startDir));
dir.setPreventDoubleWrite(false);
dir.setAllowRandomFileNotFoundException(false);
- // test uses IW unref'ed helper which is unaware of retries
- dir.setEnableVirusScanner(false);
IndexWriter modifier = new IndexWriter(dir,
newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(1000)
@@ -913,8 +908,6 @@ public class TestIndexWriterDelete extends LuceneTestCase {
String[] text = { "Amsterdam", "Venice" };
MockDirectoryWrapper dir = newMockDirectory();
- // test uses IW unref'ed helper which is unaware of retries
- dir.setEnableVirusScanner(false);
IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
modifier.commit();
dir.failOn(failure.reset());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index 055fc5a..db2253b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -916,7 +916,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
isCommit = true;
}
- if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName())) {
+ if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFiles".equals(trace[i].getMethodName())) {
isDelete = true;
}
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) {
@@ -950,7 +950,6 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
for (FailOnlyInCommit failure : failures) {
MockDirectoryWrapper dir = newMockDirectory();
dir.setFailOnCreateOutput(false);
- dir.setEnableVirusScanner(false); // we check for specific list of files
int fileCount = dir.listAll().length;
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
Document doc = new Document();
@@ -965,7 +964,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
} catch (RuntimeException re) {
// Expected
}
- assertTrue(failure.failOnCommit && failure.failOnDeleteFile);
+ assertTrue("failOnCommit=" + failure.failOnCommit + " failOnDeleteFile=" + failure.failOnDeleteFile, failure.failOnCommit && failure.failOnDeleteFile);
w.rollback();
String files[] = dir.listAll();
assertTrue(files.length == fileCount || (files.length == fileCount+1 && Arrays.asList(files).contains(IndexWriter.WRITE_LOCK_NAME)));
@@ -1178,10 +1177,6 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
public void testSimulatedCorruptIndex1() throws IOException {
BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we are corrupting it!
- if (dir instanceof MockDirectoryWrapper) {
- // we want to ensure our corruption always succeeds!
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = null;
@@ -1210,7 +1205,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
}
in.close();
out.close();
- dir.deleteFile(fileNameIn);
+ dir.deleteFiles(Collections.singleton(fileNameIn));
IndexReader reader = null;
try {
@@ -1230,10 +1225,6 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
public void testSimulatedCorruptIndex2() throws IOException {
BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we are corrupting it!
- if (dir instanceof MockDirectoryWrapper) {
- // we want to ensure our corruption always succeeds!
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = null;
writer = new IndexWriter(
@@ -1264,7 +1255,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
assertTrue(si.info.getUseCompoundFile());
List<String> victims = new ArrayList<String>(si.info.files());
Collections.shuffle(victims, random());
- dir.deleteFile(victims.get(0));
+ dir.deleteFiles(Collections.singleton(victims.get(0)));
corrupted = true;
break;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
index 19030cc..7570163 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
@@ -126,7 +126,6 @@ public class TestIndexWriterForceMerge extends LuceneTestCase {
public void testForceMergeTempSpaceUsage() throws IOException {
final MockDirectoryWrapper dir = newMockDirectory();
- dir.setEnableVirusScanner(false);
// don't use MockAnalyzer, variable length payloads can cause merge to make things bigger,
// since things are optimized for fixed length case. this is a problem for MemoryPF's encoding.
// (it might have other problems too)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterFromReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterFromReader.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterFromReader.java
index baa74e1..44d938f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterFromReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterFromReader.java
@@ -17,6 +17,10 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.AlreadyClosedException;
@@ -24,10 +28,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import org.apache.lucene.util.TestUtil;
public class TestIndexWriterFromReader extends LuceneTestCase {
@@ -111,10 +112,6 @@ public class TestIndexWriterFromReader extends LuceneTestCase {
// Pull NRT reader after writer has committed and then indexed another doc:
public void testAfterCommitThenIndex() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // We only hit exc if stale segments file was deleted:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
w.addDocument(new Document());
w.commit();
@@ -140,10 +137,6 @@ public class TestIndexWriterFromReader extends LuceneTestCase {
// NRT rollback: pull NRT reader after writer has committed and then before indexing another doc
public void testNRTRollback() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // We only hit exc if stale segments file was deleted:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
w.addDocument(new Document());
w.commit();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
index f75230c..e6fe6e8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
@@ -64,7 +64,6 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase {
System.out.println("TEST: cycle: diskFree=" + diskFree);
}
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
- dir.setEnableVirusScanner(false); // currently uses the IW unreferenced files method, unaware of retries
dir.setMaxSizeInBytes(diskFree);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
MergeScheduler ms = writer.getConfig().getMergeScheduler();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java b/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
index 6d9bb69..bb452bf 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -25,8 +26,9 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
/** LUCENE-5574 */
@SuppressFileSystems("WindowsFS") // the bug doesn't happen on windows.
@@ -40,12 +42,6 @@ public class TestNRTReaderCleanup extends LuceneTestCase {
MockDirectoryWrapper dir = newMockDirectory();
- // don't act like windows either, or the test won't simulate the condition
- dir.setEnableVirusScanner(false);
-
- // Allow deletion of still open files:
- dir.setNoDeleteOpenFile(false);
-
// Allow writing to same file more than once:
dir.setPreventDoubleWrite(false);
@@ -66,9 +62,7 @@ public class TestNRTReaderCleanup extends LuceneTestCase {
w.close();
// Blow away index and make a new writer:
- for(String fileName : dir.listAll()) {
- dir.deleteFile(fileName);
- }
+ dir.deleteFiles(Arrays.asList(dir.listAll()));
w = new RandomIndexWriter(random(), dir);
w.addDocument(doc);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestNeverDelete.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNeverDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestNeverDelete.java
index 0a0438a..3d15971 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNeverDelete.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNeverDelete.java
@@ -39,12 +39,6 @@ public class TestNeverDelete extends LuceneTestCase {
final Path tmpDir = createTempDir("TestNeverDelete");
final BaseDirectoryWrapper d = newFSDirectory(tmpDir);
- // We want to "see" files removed if Lucene removed
- // them. This is still worth running on Windows since
- // some files the IR opens and closes.
- if (d instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)d).setNoDeleteOpenFile(false);
- }
final RandomIndexWriter w = new RandomIndexWriter(random(),
d,
newIndexWriterConfig(new MockAnalyzer(random()))
@@ -107,7 +101,5 @@ public class TestNeverDelete extends LuceneTestCase {
}
w.close();
d.close();
-
- IOUtils.rm(tmpDir);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
index 1fb2206..df1ed20 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
@@ -1182,10 +1182,6 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
@Test
public void testDeleteUnusedUpdatesFiles() throws Exception {
Directory dir = newDirectory();
- // test explicitly needs files to always be actually deleted
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
index 551a073..ac32bab 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
@@ -189,10 +189,7 @@ public class TestOmitPositions extends LuceneTestCase {
// Verifies no *.prx exists when all fields omit term positions:
public void testNoPrxFile() throws Throwable {
Directory ram = newDirectory();
- if (ram instanceof MockDirectoryWrapper) {
- // we verify some files get deleted
- ((MockDirectoryWrapper)ram).setEnableVirusScanner(false);
- }
+
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
.setMaxBufferedDocs(3)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
index 187216b..6e8874f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
@@ -40,6 +40,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
public class TestOmitTf extends LuceneTestCase {
@@ -219,10 +220,6 @@ public class TestOmitTf extends LuceneTestCase {
// Verifies no *.prx exists when all fields omit term freq:
public void testNoPrxFile() throws Throwable {
Directory ram = newDirectory();
- if (ram instanceof MockDirectoryWrapper) {
- // we verify some files get deleted
- ((MockDirectoryWrapper)ram).setEnableVirusScanner(false);
- }
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
.setMaxBufferedDocs(3)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
index c94b66a..360933a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -50,7 +51,6 @@ public class TestPersistentSnapshotDeletionPolicy extends TestSnapshotDeletionPo
public void testExistingSnapshots() throws Exception {
int numSnapshots = 3;
MockDirectoryWrapper dir = newMockDirectory();
- dir.setEnableVirusScanner(false); // test relies on files actually being deleted
IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy(dir)));
PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
assertNull(psdp.getLastSaveFile());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
index f367024..a0fda68 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
@@ -39,10 +39,6 @@ public class TestRollingUpdates extends LuceneTestCase {
public void testRollingUpdates() throws Exception {
Random random = new Random(random().nextLong());
final BaseDirectoryWrapper dir = newDirectory();
- // test checks for no unref'ed files with the IW helper method, which isn't aware of "tried to delete files"
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
final LineFileDocs docs = new LineFileDocs(random, true);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
index 1780494..e6c8f57 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
@@ -32,6 +32,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.ThreadInterruptedException;
import org.junit.Test;
@@ -103,10 +104,6 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
}
private void runTest(Random random, Directory dir) throws Exception {
- // we use the IW unref'ed files check which is unaware of retries:
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
// Run for ~1 seconds
final long stopTime = System.currentTimeMillis() + 1000;
@@ -257,10 +254,6 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
// Create 3 snapshots: snapshot0, snapshot1, snapshot2
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // we verify some files get deleted
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy()));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
prepareIndexAndSnapshots(sdp, writer, numSnapshots);
@@ -285,10 +278,7 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
@Test
public void testMultiThreadedSnapshotting() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test relies on files actually being deleted
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
+
final IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy()));
final SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
@@ -364,10 +354,6 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
@Test
public void testReleaseSnapshot() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // we rely upon existence of files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy()));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
prepareIndexAndSnapshots(sdp, writer, 1);
@@ -417,10 +403,6 @@ public class TestSnapshotDeletionPolicy extends LuceneTestCase {
// Tests the behavior of SDP when commits that are given at ctor are missing
// on onInit().
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // we rely upon existence of files
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy()));
SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
writer.addDocument(new Document());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
index c412545..f1d0418 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
@@ -43,15 +43,6 @@ public class TestSwappedIndexFiles extends LuceneTestCase {
Directory dir1 = newDirectory();
Directory dir2 = newDirectory();
- if (dir1 instanceof MockDirectoryWrapper) {
- // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
- ((MockDirectoryWrapper) dir1).setEnableVirusScanner(false);
- }
- if (dir2 instanceof MockDirectoryWrapper) {
- // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
- ((MockDirectoryWrapper) dir2).setEnableVirusScanner(false);
- }
-
// Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too:
boolean useCFS = random().nextInt(5) == 1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index 15d4a06..2f6f38a 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -175,7 +175,7 @@ public class TestPointQueries extends LuceneTestCase {
iwc.setCodec(getCodec());
Directory dir;
if (values.length > 100000) {
- dir = noVirusChecker(newFSDirectory(createTempDir("TestRangeTree")));
+ dir = newFSDirectory(createTempDir("TestRangeTree"));
} else {
dir = getDirectory();
}
@@ -439,7 +439,7 @@ public class TestPointQueries extends LuceneTestCase {
Directory dir;
if (docValues.length > 100000) {
- dir = noVirusChecker(newFSDirectory(createTempDir("TestPointRangeQuery")));
+ dir = newFSDirectory(createTempDir("TestPointQueries"));
} else {
dir = getDirectory();
}
@@ -1018,15 +1018,8 @@ public class TestPointQueries extends LuceneTestCase {
IOUtils.close(w, dir);
}
- private static Directory noVirusChecker(Directory dir) {
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
- }
-
private static Directory getDirectory() {
- return noVirusChecker(newDirectory());
+ return newDirectory();
}
private static Codec getCodec() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
index ea2ec7c..3df8dc5 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
@@ -211,58 +211,54 @@ public class TestBufferedIndexInput extends LuceneTestCase {
public void testSetBufferSize() throws IOException {
Path indexDir = createTempDir("testSetBufferSize");
MockFSDirectory dir = new MockFSDirectory(indexDir, random());
- try {
- IndexWriter writer = new IndexWriter(
- dir,
- new IndexWriterConfig(new MockAnalyzer(random())).
- setOpenMode(OpenMode.CREATE).
- setMergePolicy(newLogMergePolicy(false))
- );
- for(int i=0;i<37;i++) {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa bbb ccc ddd" + i, Field.Store.YES));
- doc.add(newTextField("id", "" + i, Field.Store.YES));
- writer.addDocument(doc);
- }
+ IndexWriter writer = new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random())).
+ setOpenMode(OpenMode.CREATE).
+ setMergePolicy(newLogMergePolicy(false))
+ );
+ for(int i=0;i<37;i++) {
+ Document doc = new Document();
+ doc.add(newTextField("content", "aaa bbb ccc ddd" + i, Field.Store.YES));
+ doc.add(newTextField("id", "" + i, Field.Store.YES));
+ writer.addDocument(doc);
+ }
- dir.allIndexInputs.clear();
+ dir.allIndexInputs.clear();
- IndexReader reader = DirectoryReader.open(writer);
- Term aaa = new Term("content", "aaa");
- Term bbb = new Term("content", "bbb");
+ IndexReader reader = DirectoryReader.open(writer);
+ Term aaa = new Term("content", "aaa");
+ Term bbb = new Term("content", "bbb");
- reader.close();
+ reader.close();
- dir.tweakBufferSizes();
- writer.deleteDocuments(new Term("id", "0"));
- reader = DirectoryReader.open(writer);
- IndexSearcher searcher = newSearcher(reader);
- ScoreDoc[] hits = searcher.search(new TermQuery(bbb), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(36, hits.length);
+ dir.tweakBufferSizes();
+ writer.deleteDocuments(new Term("id", "0"));
+ reader = DirectoryReader.open(writer);
+ IndexSearcher searcher = newSearcher(reader);
+ ScoreDoc[] hits = searcher.search(new TermQuery(bbb), 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(36, hits.length);
- reader.close();
+ reader.close();
- dir.tweakBufferSizes();
- writer.deleteDocuments(new Term("id", "4"));
- reader = DirectoryReader.open(writer);
- searcher = newSearcher(reader);
+ dir.tweakBufferSizes();
+ writer.deleteDocuments(new Term("id", "4"));
+ reader = DirectoryReader.open(writer);
+ searcher = newSearcher(reader);
- hits = searcher.search(new TermQuery(bbb), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(35, hits.length);
- dir.tweakBufferSizes();
- hits = searcher.search(new TermQuery(new Term("id", "33")), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(1, hits.length);
- hits = searcher.search(new TermQuery(aaa), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(35, hits.length);
- writer.close();
- reader.close();
- } finally {
- IOUtils.rm(indexDir);
- }
+ hits = searcher.search(new TermQuery(bbb), 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(35, hits.length);
+ dir.tweakBufferSizes();
+ hits = searcher.search(new TermQuery(new Term("id", "33")), 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(1, hits.length);
+ hits = searcher.search(new TermQuery(aaa), 1000).scoreDocs;
+ dir.tweakBufferSizes();
+ assertEquals(35, hits.length);
+ writer.close();
+ reader.close();
}
private static class MockFSDirectory extends FilterDirectory {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
index 8534fbf..3b65389 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
@@ -21,10 +21,12 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
public class TestDirectory extends LuceneTestCase {
@@ -32,6 +34,7 @@ public class TestDirectory extends LuceneTestCase {
// path, can read, write, and lock files.
public void testDirectInstantiation() throws Exception {
final Path path = createTempDir("testDirectInstantiation");
+ assumeFalse("test deletes files through different FSDir instances", TestUtil.hasVirusChecker(path));
final byte[] largeBuffer = new byte[random().nextInt(256*1024)], largeReadBuffer = new byte[largeBuffer.length];
for (int i = 0; i < largeBuffer.length; i++) {
@@ -79,7 +82,7 @@ public class TestDirectory extends LuceneTestCase {
}
// delete with a different dir
- dirs[(i+1)%dirs.length].deleteFile(fname);
+ dirs[(i+1)%dirs.length].deleteFiles(Collections.singleton(fname));
for (int j=0; j<dirs.length; j++) {
FSDirectory d2 = dirs[j];
@@ -110,23 +113,17 @@ public class TestDirectory extends LuceneTestCase {
dir.close();
assertFalse(dir.isOpen);
}
-
- IOUtils.rm(path);
}
// LUCENE-1468
@SuppressWarnings("resource")
public void testCopySubdir() throws Throwable {
Path path = createTempDir("testsubdir");
- try {
- Files.createDirectory(path.resolve("subdir"));
- FSDirectory fsDir = new SimpleFSDirectory(path);
- RAMDirectory ramDir = new RAMDirectory(fsDir, newIOContext(random()));
- List<String> files = Arrays.asList(ramDir.listAll());
- assertFalse(files.contains("subdir"));
- } finally {
- IOUtils.rm(path);
- }
+ Files.createDirectory(path.resolve("subdir"));
+ FSDirectory fsDir = new SimpleFSDirectory(path);
+ RAMDirectory ramDir = new RAMDirectory(fsDir, newIOContext(random()));
+ List<String> files = Arrays.asList(ramDir.listAll());
+ assertFalse(files.contains("subdir"));
}
// LUCENE-1468
@@ -145,7 +142,6 @@ public class TestDirectory extends LuceneTestCase {
}
} finally {
fsDir.close();
- IOUtils.rm(path);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
index 60db126..04de4b2 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestFileSwitchDirectory.java
@@ -100,7 +100,6 @@ public class TestFileSwitchDirectory extends BaseDirectoryTestCase {
public void testNoDir() throws Throwable {
Path primDir = createTempDir("foo");
Path secondDir = createTempDir("bar");
- IOUtils.rm(primDir, secondDir);
Directory dir = newFSSwitchDirectory(primDir, secondDir, Collections.<String>emptySet());
try {
DirectoryReader.open(dir);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java b/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
index b53707e..71284f1 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
@@ -20,8 +20,10 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Collections;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
/** Simple tests for NativeFSLockFactory */
public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
@@ -79,19 +81,15 @@ public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
/** delete the lockfile and test ensureValid fails */
public void testDeleteLockFile() throws IOException {
- Directory dir = getDirectory(createTempDir());
- try {
+ try (Directory dir = getDirectory(createTempDir())) {
+ assumeFalse("we must be able to delete an open file", TestUtil.hasWindowsFS(dir));
+ assumeFalse("we must be able to delete an open file", TestUtil.hasVirusChecker(dir));
+
Lock lock = dir.obtainLock("test.lock");
lock.ensureValid();
-
- try {
- dir.deleteFile("test.lock");
- } catch (Exception e) {
- // we can't delete a file for some reason, just clean up and assume the test.
- IOUtils.closeWhileHandlingException(lock);
- assumeNoException("test requires the ability to delete a locked file", e);
- }
-
+
+ dir.deleteFiles(Collections.singleton("test.lock"));
+
try {
lock.ensureValid();
fail("no exception");
@@ -100,9 +98,6 @@ public class TestNativeFSLockFactory extends BaseLockFactoryTestCase {
} finally {
IOUtils.closeWhileHandlingException(lock);
}
- } finally {
- // Do this in finally clause in case the assumeNoException is false:
- dir.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
index 70cd054..dd4e175 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
@@ -83,7 +83,6 @@ public class TestRAMDirectory extends BaseDirectoryTestCase {
assertFalse(files.contains("subdir"));
} finally {
IOUtils.close(fsDir);
- IOUtils.rm(path);
}
}
[5/5] lucene-solr git commit: fix a few tests; add some nocommits
Posted by mi...@apache.org.
fix a few tests; add some nocommits
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4a2bf2b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4a2bf2b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4a2bf2b
Branch: refs/heads/lucene-6835
Commit: b4a2bf2b41952be6ec3027943425d15365bcd9a0
Parents: 84f4458
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Feb 2 18:07:06 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Feb 2 18:07:06 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/lucene/store/Directory.java | 1 +
.../org/apache/lucene/store/FSDirectory.java | 55 ++++++++++++++------
.../lucene/index/TestDirectoryReader.java | 6 ++-
.../org/apache/lucene/util/TestIOUtils.java | 3 ++
.../apache/lucene/mockfile/VirusCheckingFS.java | 4 ++
5 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/java/org/apache/lucene/store/Directory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index b9e5ad4..bb12c92 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -47,6 +47,7 @@ public abstract class Directory implements Closeable {
*
* @throws IOException in case of IO error
*/
+ // nocommit should this sort?
public abstract String[] listAll() throws IOException;
/** Removes the specified files from the directory. If an exception is thrown, behavior is undefined
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index 0e1d4e9..9eb78b4 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -229,22 +229,20 @@ public abstract class FSDirectory extends BaseDirectory {
return listAll(directory, pendingDeletes);
}
- /** Returns the length in bytes of a file in the directory. */
@Override
public long fileLength(String name) throws IOException {
ensureOpen();
return Files.size(directory.resolve(name));
}
- /** Removes an existing file in the directory. */
@Override
public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
+ // nocommit isn't it an error if they were already pending delete?
pendingDeletes.addAll(names);
deletePendingFiles();
}
- /** Creates an IndexOutput for the file with the given name. */
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
@@ -277,7 +275,7 @@ public abstract class FSDirectory extends BaseDirectory {
protected void ensureCanRead(String name) throws IOException {
deletePendingFiles();
if (pendingDeletes.contains(name)) {
- throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be overwritten");
+ throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be opened for read");
}
}
@@ -299,7 +297,6 @@ public abstract class FSDirectory extends BaseDirectory {
IOUtils.fsync(directory, true);
}
- /** Closes the store to future operations. */
@Override
public synchronized void close() throws IOException {
isOpen = false;
@@ -312,7 +309,6 @@ public abstract class FSDirectory extends BaseDirectory {
return directory;
}
- /** For debug output. */
@Override
public String toString() {
return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
@@ -324,14 +320,13 @@ public abstract class FSDirectory extends BaseDirectory {
}
/** Returns true if the file was successfully removed. */
- private boolean deleteFile(String name) throws IOException {
+ private synchronized boolean deleteFile(String name) throws IOException {
+ pendingDeletes.remove(name);
try {
Files.delete(directory.resolve(name));
- pendingDeletes.remove(name);
return true;
} catch (NoSuchFileException | FileNotFoundException e) {
// We were asked to delete a non-existent file:
- pendingDeletes.remove(name);
throw e;
} catch (IOException ioe) {
// On windows, a file delete can fail because there's still an open
@@ -358,21 +353,36 @@ public abstract class FSDirectory extends BaseDirectory {
}
/** Try to delete any pending files that we had previously tried to delete but failed
- * because we are on Windows and the files were still
- * held open. */
- public void deletePendingFiles() throws IOException {
+ * because we are on Windows and the files were still held open. */
+ public synchronized void deletePendingFiles() throws IOException {
// TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?
// Clone the set because it will change as we iterate:
List<String> toDelete = new ArrayList<>(pendingDeletes);
+ System.out.println("del pending: " + pendingDeletes);
// First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
// before we remove any files they reference. If any delete of segments_N fails, we leave all other files
// undeleted so index is never in a corrupt state:
+ Throwable firstException = null;
for (String fileName : toDelete) {
if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
- if (deleteFile(fileName) == false) {
- return;
+ try {
+ if (deleteFile(fileName) == false) {
+ // nocommit
+ System.out.println(" false on " + fileName + "; skipping the rest");
+ return;
+ }
+ } catch (Throwable t) {
+ if (firstException == null) {
+ firstException = t;
+ } else {
+ firstException.addSuppressed(t);
+ }
+ // nocommit
+ System.out.println(" fail on " + fileName + ":");
+ t.printStackTrace(System.out);
+ throw t;
}
}
}
@@ -381,9 +391,24 @@ public abstract class FSDirectory extends BaseDirectory {
// leave a corrupt commit in the index even in the presense of virus checkers:
for(String fileName : toDelete) {
if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
- deleteFile(fileName);
+ try {
+ deleteFile(fileName);
+ } catch (Throwable t) {
+ if (firstException == null) {
+ firstException = t;
+ } else {
+ firstException.addSuppressed(t);
+ }
+ // nocommit
+ System.out.println(" fail on " + fileName + ":");
+ t.printStackTrace(System.out);
+ throw t;
+ }
}
}
+
+ // Does nothing if firstException is null:
+ IOUtils.reThrow(firstException);
}
final class FSIndexOutput extends OutputStreamIndexOutput {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 2c4f392..2213033 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -472,7 +472,10 @@ void assertTermDocsCount(String msg,
public void testOpenReaderAfterDelete() throws IOException {
Path dirFile = createTempDir("deletetest");
Directory dir = newFSDirectory(dirFile);
- assumeFalse("test deletes files directly", TestUtil.hasVirusChecker(dir));
+ if (TestUtil.hasVirusChecker(dir)) {
+ dir.close();
+ assumeTrue("test deletes files directly", false);
+ }
if (dir instanceof BaseDirectoryWrapper) {
((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we will hit NoSuchFileException in MDW since we nuked it!
}
@@ -1055,7 +1058,6 @@ void assertTermDocsCount(String msg,
public void testIndexExistsOnNonExistentDirectory() throws Exception {
Path tempDir = createTempDir("testIndexExistsOnNonExistentDirectory");
- IOUtils.rm(tempDir);
Directory dir = newFSDirectory(tempDir);
assertFalse(DirectoryReader.indexExists(dir));
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
index 7a112cd..726f4f8 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
@@ -38,8 +38,11 @@ import java.util.UUID;
import org.apache.lucene.mockfile.FilterFileSystem;
import org.apache.lucene.mockfile.FilterFileSystemProvider;
import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.mockfile.VirusCheckingFS;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
/** Simple test methods for IOUtils */
+@SuppressFileSystems("VirusCheckingFS")
public class TestIOUtils extends LuceneTestCase {
public void testDeleteFileIgnoringExceptions() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
index 801a688..b30ce94 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
@@ -25,6 +25,7 @@ import java.nio.file.Path;
import java.util.Random;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.util.LuceneTestCase;
/**
* Acts like Windows, where random programs may open the files you just wrote in an unfriendly
@@ -56,6 +57,9 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
&& Files.exists(path) // important that we NOT delay a NoSuchFileException until later
&& path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
&& random.nextInt(5) == 1) {
+ if (true || LuceneTestCase.VERBOSE) {
+ System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
+ }
throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");
}
super.delete(path);
[2/5] lucene-solr git commit: migrate current patch from svn
Posted by mi...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java b/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
index f05297e..793c005 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
@@ -19,8 +19,10 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.Collections;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
/** Simple tests for SimpleFSLockFactory */
public class TestSimpleFSLockFactory extends BaseLockFactoryTestCase {
@@ -34,11 +36,12 @@ public class TestSimpleFSLockFactory extends BaseLockFactoryTestCase {
public void testDeleteLockFile() throws IOException {
Directory dir = getDirectory(createTempDir());
try {
+ assumeFalse("test directly deletes lock files", TestUtil.hasVirusChecker(dir));
Lock lock = dir.obtainLock("test.lock");
lock.ensureValid();
try {
- dir.deleteFile("test.lock");
+ dir.deleteFiles(Collections.singleton("test.lock"));
} catch (Exception e) {
// we can't delete a file for some reason, just clean up and assume the test.
IOUtils.closeWhileHandlingException(lock);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java b/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
index 0985587..f9551d6 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
@@ -43,7 +43,7 @@ public class TestTrackingDirectoryWrapper extends BaseDirectoryTestCase {
TrackingDirectoryWrapper dir = new TrackingDirectoryWrapper(new RAMDirectory());
dir.createOutput("foo", newIOContext(random())).close();
assertEquals(asSet("foo"), dir.getCreatedFiles());
- dir.deleteFile("foo");
+ dir.deleteFiles(Collections.singleton("foo"));
assertEquals(Collections.emptySet(), dir.getCreatedFiles());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
index d8a878c..ca8e802 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
@@ -30,6 +30,7 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.OfflineSorter.BufferSize;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
import org.apache.lucene.util.OfflineSorter.SortInfo;
@@ -37,6 +38,7 @@ import org.apache.lucene.util.OfflineSorter.SortInfo;
/**
* Tests for on-disk merge sorting.
*/
+@SuppressFileSystems("VirusCheckingFS")
public class TestOfflineSorter extends LuceneTestCase {
private Path tempDir;
@@ -54,30 +56,14 @@ public class TestOfflineSorter extends LuceneTestCase {
super.tearDown();
}
- private static Directory newDirectoryNoVirusScanner() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
- }
-
- private static Directory newFSDirectoryNoVirusScanner() {
- Directory dir = newFSDirectory(createTempDir());
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
- }
-
public void testEmpty() throws Exception {
- try (Directory dir = newDirectoryNoVirusScanner()) {
+ try (Directory dir = newDirectory()) {
checkSort(dir, new OfflineSorter(dir, "foo"), new byte [][] {});
}
}
public void testSingleLine() throws Exception {
- try (Directory dir = newDirectoryNoVirusScanner()) {
+ try (Directory dir = newDirectory()) {
checkSort(dir, new OfflineSorter(dir, "foo"), new byte [][] {
"Single line only.".getBytes(StandardCharsets.UTF_8)
});
@@ -86,7 +72,7 @@ public class TestOfflineSorter extends LuceneTestCase {
public void testIntermediateMerges() throws Exception {
// Sort 20 mb worth of data with 1mb buffer, binary merging.
- try (Directory dir = newDirectoryNoVirusScanner()) {
+ try (Directory dir = newDirectory()) {
SortInfo info = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 2),
generateRandom((int)OfflineSorter.MB * 20));
assertTrue(info.mergeRounds > 10);
@@ -95,7 +81,7 @@ public class TestOfflineSorter extends LuceneTestCase {
public void testSmallRandom() throws Exception {
// Sort 20 mb worth of data with 1mb buffer.
- try (Directory dir = newDirectoryNoVirusScanner()) {
+ try (Directory dir = newDirectory()) {
SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES),
generateRandom((int)OfflineSorter.MB * 20));
assertEquals(1, sortInfo.mergeRounds);
@@ -105,7 +91,7 @@ public class TestOfflineSorter extends LuceneTestCase {
@Nightly
public void testLargerRandom() throws Exception {
// Sort 100MB worth of data with 15mb buffer.
- try (Directory dir = newFSDirectoryNoVirusScanner()) {
+ try (Directory dir = newFSDirectory(createTempDir())) {
checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.MAX_TEMPFILES),
generateRandom((int)OfflineSorter.MB * 100));
}
@@ -223,7 +209,7 @@ public class TestOfflineSorter extends LuceneTestCase {
Thread[] threads = new Thread[TestUtil.nextInt(random(), 4, 10)];
final AtomicBoolean failed = new AtomicBoolean();
final int iters = atLeast(1000);
- try (Directory dir = newDirectoryNoVirusScanner()) {
+ try (Directory dir = newDirectory()) {
for(int i=0;i<threads.length;i++) {
final int threadID = i;
threads[i] = new Thread() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index 3448150..dc17f96 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -22,6 +22,7 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.index.PointValues.IntersectVisitor;
@@ -394,9 +395,6 @@ public class TestBKD extends LuceneTestCase {
try {
dir.setRandomIOExceptionRate(0.05);
dir.setRandomIOExceptionRateOnOpen(0.05);
- if (dir instanceof MockDirectoryWrapper) {
- dir.setEnableVirusScanner(false);
- }
verify(dir, docValues, null, numDims, numBytesPerDim, 50, maxMBHeap);
} catch (IllegalArgumentException iae) {
// This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry w/ more heap
@@ -820,9 +818,9 @@ public class TestBKD extends LuceneTestCase {
}
}
in.close();
- dir.deleteFile("bkd");
+ dir.deleteFiles(Collections.singleton("bkd"));
if (toMerge != null) {
- dir.deleteFile("bkd2");
+ dir.deleteFiles(Collections.singleton("bkd2"));
}
success = true;
} finally {
@@ -848,9 +846,6 @@ public class TestBKD extends LuceneTestCase {
} else {
dir = newDirectory();
}
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
return dir;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
index 57d8374..e3868e4 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -18,6 +18,7 @@ package org.apache.lucene.util.fst;
*/
import java.util.Arrays;
+import java.util.Collections;
import java.util.Random;
import org.apache.lucene.store.Directory;
@@ -31,6 +32,7 @@ import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util.packed.PackedInts;
import org.junit.Ignore;
+
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
@Ignore("Requires tons of heap to run (30 GB hits OOME but 35 GB passes after ~4.5 hours)")
@@ -126,7 +128,7 @@ public class Test2BFST extends LuceneTestCase {
fst = new FST<>(in, outputs);
in.close();
} else {
- dir.deleteFile("fst");
+ dir.deleteFiles(Collections.singleton("fst"));
}
}
}
@@ -203,7 +205,7 @@ public class Test2BFST extends LuceneTestCase {
fst = new FST<>(in, outputs);
in.close();
} else {
- dir.deleteFile("fst");
+ dir.deleteFiles(Collections.singleton("fst"));
}
}
}
@@ -287,7 +289,7 @@ public class Test2BFST extends LuceneTestCase {
fst = new FST<>(in, outputs);
in.close();
} else {
- dir.deleteFile("fst");
+ dir.deleteFiles(Collections.singleton("fst"));
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
index 27dd169..ac948d7 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -66,10 +66,11 @@ import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.FST.Arc;
@@ -83,6 +84,7 @@ import static org.apache.lucene.util.fst.FSTTester.simpleRandomString;
import static org.apache.lucene.util.fst.FSTTester.toIntsRef;
@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
+@SuppressFileSystems({ "VirusCheckingFS" })
@Slow
public class TestFSTs extends LuceneTestCase {
@@ -93,7 +95,6 @@ public class TestFSTs extends LuceneTestCase {
super.setUp();
dir = newMockDirectory();
dir.setPreventDoubleWrite(false);
- dir.setEnableVirusScanner(false);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
index bab144b..e11cfef 100644
--- a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
+++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import java.nio.LongBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Random;
@@ -840,7 +841,7 @@ public class TestPackedInts extends LuceneTestCase {
assertEquals(mutable.get(i), reader.get(i));
}
in.close();
- directory.deleteFile("packed-ints.bin");
+ directory.deleteFiles(Collections.singleton("packed-ints.bin"));
}
directory.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java b/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
index 9ffa011..970d2ed 100644
--- a/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
+++ b/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
@@ -46,7 +46,6 @@ public class TestFSTsMisc extends LuceneTestCase {
super.setUp();
dir = newMockDirectory();
dir.setPreventDoubleWrite(false);
- dir.setEnableVirusScanner(false);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
----------------------------------------------------------------------
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
index 310399f..51ebae6 100644
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Path;
+import java.util.Collections;
import java.util.HashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -424,22 +425,6 @@ public class IndexAndTaxonomyReplicationClientTest extends ReplicatorTestCase {
}
}
- if (indexStatus == null || indexStatus.clean == false) {
-
- // Because segments file for taxo index is replicated after
- // main index's segments file, if there's an error while replicating
- // main index's segments file and if virus checker prevents
- // deletion of taxo index's segments file, it can look like corruption.
- // But it should be "false" meaning if we remove the latest segments
- // file then the index is intact. It's like pulling a hideous
- // looking rock out of the ground, but then you pull the cruft
- // off the outside of it and discover it's actually a beautiful
- // diamond:
- String segmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(handlerTaxoDir);
- assertTrue(handlerTaxoDir.didTryToDelete(segmentsFileName));
- handlerTaxoDir.getDelegate().deleteFile(segmentsFileName);
- TestUtil.checkIndex(handlerTaxoDir.getDelegate());
- }
} catch (IOException e) {
failed.set(true);
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
----------------------------------------------------------------------
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
index 2c832da..7a4465a 100644
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
@@ -77,10 +77,6 @@ public class IndexAndTaxonomyRevisionTest extends ReplicatorTestCase {
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
- // we look to see that certain files are deleted:
- if (indexDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)indexDir).setEnableVirusScanner(false);
- }
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
@@ -101,10 +97,6 @@ public class IndexAndTaxonomyRevisionTest extends ReplicatorTestCase {
indexWriter.close();
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
- if (indexDir instanceof MockDirectoryWrapper) {
- // set back to on for other tests
- ((MockDirectoryWrapper)indexDir).setEnableVirusScanner(true);
- }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java
----------------------------------------------------------------------
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java
index 6ebdd15..9e267ab 100644
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java
+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java
@@ -73,10 +73,6 @@ public class IndexRevisionTest extends ReplicatorTestCase {
@Test
public void testRevisionRelease() throws Exception {
Directory dir = newDirectory();
- // we look to see that certain files are deleted:
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig conf = new IndexWriterConfig(null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
----------------------------------------------------------------------
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java b/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
index 3c9a8d1..b092e83 100644
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
@@ -119,57 +119,35 @@ public class LocalReplicatorTest extends ReplicatorTestCase {
@Test
public void testPublishSameRevision() throws IOException {
- // we look to see that certain files are deleted:
- if (sourceDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)sourceDir).setEnableVirusScanner(false);
- }
- try {
- Revision rev = createRevision(1);
- replicator.publish(rev);
- SessionToken res = replicator.checkForUpdate(null);
- assertNotNull(res);
- assertEquals(rev.getVersion(), res.version);
- replicator.release(res.id);
- replicator.publish(new IndexRevision(sourceWriter));
- res = replicator.checkForUpdate(res.version);
- assertNull(res);
+ Revision rev = createRevision(1);
+ replicator.publish(rev);
+ SessionToken res = replicator.checkForUpdate(null);
+ assertNotNull(res);
+ assertEquals(rev.getVersion(), res.version);
+ replicator.release(res.id);
+ replicator.publish(new IndexRevision(sourceWriter));
+ res = replicator.checkForUpdate(res.version);
+ assertNull(res);
- // now make sure that publishing same revision doesn't leave revisions
- // "locked", i.e. that replicator releases revisions even when they are not
- // kept
- replicator.publish(createRevision(2));
- assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
- } finally {
- if (sourceDir instanceof MockDirectoryWrapper) {
- // set back to on for other tests
- ((MockDirectoryWrapper)sourceDir).setEnableVirusScanner(true);
- }
- }
+ // now make sure that publishing same revision doesn't leave revisions
+ // "locked", i.e. that replicator releases revisions even when they are not
+ // kept
+ replicator.publish(createRevision(2));
+ assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
}
@Test
public void testPublishOlderRev() throws IOException {
- // we look to see that certain files are deleted:
- if (sourceDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)sourceDir).setEnableVirusScanner(false);
- }
+ replicator.publish(createRevision(1));
+ Revision old = new IndexRevision(sourceWriter);
+ replicator.publish(createRevision(2));
try {
- replicator.publish(createRevision(1));
- Revision old = new IndexRevision(sourceWriter);
- replicator.publish(createRevision(2));
- try {
- replicator.publish(old);
- fail("should have failed to publish an older revision");
- } catch (IllegalArgumentException e) {
- // expected
- }
- assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
- } finally {
- if (sourceDir instanceof MockDirectoryWrapper) {
- // set back to on for other tests
- ((MockDirectoryWrapper)sourceDir).setEnableVirusScanner(true);
- }
+ replicator.publish(old);
+ fail("should have failed to publish an older revision");
+ } catch (IllegalArgumentException e) {
+ // expected
}
+ assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
}
@Test
@@ -210,24 +188,12 @@ public class LocalReplicatorTest extends ReplicatorTestCase {
@Test
public void testRevisionRelease() throws Exception {
- // we look to see that certain files are deleted:
- if (sourceDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)sourceDir).setEnableVirusScanner(false);
- }
-
- try {
- replicator.publish(createRevision(1));
- assertTrue(slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_1"));
- replicator.publish(createRevision(2));
- // now the files of revision 1 can be deleted
- assertTrue(slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_2"));
- assertFalse("segments_1 should not be found in index directory after revision is released", slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_1"));
- } finally {
- if (sourceDir instanceof MockDirectoryWrapper) {
- // set back to on for other tests
- ((MockDirectoryWrapper)sourceDir).setEnableVirusScanner(true);
- }
- }
+ replicator.publish(createRevision(1));
+ assertTrue(slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_1"));
+ replicator.publish(createRevision(2));
+ // now the files of revision 1 can be deleted
+ assertTrue(slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_2"));
+ assertFalse("segments_1 should not be found in index directory after revision is released", slowFileExists(sourceDir, IndexFileNames.SEGMENTS + "_1"));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java
index 42adeab..fa9ea00 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/util/BaseGeoPointTestCase.java
@@ -213,7 +213,6 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
double[] lats = new double[2*numPoints];
double[] lons = new double[2*numPoints];
Directory dir = newDirectory();
- noVirusChecker(dir);
IndexWriterConfig iwc = newIndexWriterConfig();
initIndexWriterConfig(FIELD_NAME, iwc);
@@ -560,7 +559,6 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
} else {
dir = newDirectory();
}
- noVirusChecker(dir);
Set<Integer> deleted = new HashSet<>();
// RandomIndexWriter is too slow here:
@@ -755,11 +753,4 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
IOUtils.close(r, dir);
assertFalse(failed.get());
}
-
- protected Directory noVirusChecker(Directory dir) {
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
index 0b7a5de..edf2a1a 100644
--- a/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/geo3d/TestGeo3DPoint.java
@@ -651,7 +651,7 @@ public class TestGeo3DPoint extends LuceneTestCase {
iwc.setCodec(getCodec());
Directory dir;
if (lats.length > 100000) {
- dir = noVirusChecker(newFSDirectory(createTempDir("TestBKDTree")));
+ dir = newFSDirectory(createTempDir("TestBKDTree"));
} else {
dir = getDirectory();
}
@@ -789,14 +789,7 @@ public class TestGeo3DPoint extends LuceneTestCase {
IOUtils.close(r, dir);
}
- private static Directory noVirusChecker(Directory dir) {
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
- }
-
private static Directory getDirectory() {
- return noVirusChecker(newDirectory());
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
index f54ad03..cb2d38c 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
@@ -487,7 +487,7 @@ public class AnalyzingSuggester extends Lookup implements Accountable {
tempSortedFileName = sorter.sort(tempInput.getName());
// Free disk space:
- tempDir.deleteFile(tempInput.getName());
+ tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
reader = new OfflineSorter.ByteSequencesReader(tempDir.openInput(tempSortedFileName, IOContext.READONCE));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
index e07a68c..132c630 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/ExternalRefSorter.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search.suggest.fst;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Collections;
import java.util.Comparator;
import org.apache.lucene.store.IOContext;
@@ -68,7 +69,7 @@ public class ExternalRefSorter implements BytesRefSorter, Closeable {
success = true;
} finally {
if (success) {
- sorter.getDirectory().deleteFile(input.getName());
+ sorter.getDirectory().deleteFiles(Collections.singleton(input.getName()));
} else {
IOUtils.deleteFilesIgnoringExceptions(sorter.getDirectory(), input.getName());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
index 24dd0f9..d178aa4 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
@@ -204,7 +204,7 @@ public class FSTCompletionLookup extends Lookup implements Accountable {
// We don't know the distribution of scores and we need to bucket them, so we'll sort
// and divide into equal buckets.
tempSortedFileName = sorter.sort(tempInput.getName());
- tempDir.deleteFile(tempInput.getName());
+ tempDir.deleteFiles(Collections.singleton(tempInput.getName()));
FSTCompletionBuilder builder = new FSTCompletionBuilder(
buckets, externalSorter, sharedTailLength);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
index 35c5f25..ea0394b 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
@@ -61,11 +61,7 @@ public class PersistenceTest extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
private void runTest(Class<? extends Lookup> lookupClass, boolean supportsExactWeights) throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
index 55e70bd..050593e 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
@@ -171,10 +171,6 @@ public class TestInputIterator extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
index a9c7acc..1d0891b 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
@@ -1268,10 +1268,6 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
index 47c19e1..9fbbba3 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
@@ -1213,10 +1213,6 @@ public class FuzzySuggesterTest extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
index ce332ce..7dec5fb 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
@@ -31,9 +31,6 @@ public class BytesRefSortersTest extends LuceneTestCase {
@Test
public void testExternalRefSorter() throws Exception {
Directory tempDir = newDirectory();
- if (tempDir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) tempDir).setEnableVirusScanner(false);
- }
ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter(tempDir, "temp"));
check(s);
IOUtils.close(s, tempDir);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
index bddb306..63d108c 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
@@ -273,10 +273,6 @@ public class FSTCompletionTest extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
index c213c4b..2c53ac4 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
@@ -236,10 +236,6 @@ public class WFSTCompletionTest extends LuceneTestCase {
}
private Directory getDirectory() {
- Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
+ return newDirectory();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
index 9122e99..0c40bb3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
@@ -183,11 +183,6 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
// Just tests that we can open all files returned by listAll
public void testListAll() throws Exception {
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // test lists files manually and tries to verify every .cfs it finds,
- // but a virus scanner could leave some trash.
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
// riw should sometimes create docvalues fields, etc
RandomIndexWriter riw = new RandomIndexWriter(random(), dir);
Document doc = new Document();
@@ -252,7 +247,7 @@ public abstract class BaseCompoundFormatTestCase extends BaseIndexFileFormatTest
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
- cfs.deleteFile(testfile);
+ cfs.deleteFiles(Collections.singleton(testfile));
fail("didn't get expected exception");
} catch (UnsupportedOperationException expected) {
// expected UOE
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index c53293c..9409c47 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -197,11 +197,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
public void testMergeStability() throws Exception {
assumeTrue("merge is not stable", mergeIsStable());
Directory dir = newDirectory();
- if (dir instanceof MockDirectoryWrapper) {
- // Else, the virus checker may prevent deletion of files and cause
- // us to see too many bytes used by extension in the end:
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
+
// do not use newMergePolicy that might return a MockMergePolicy that ignores the no-CFS ratio
// do not use RIW which will change things up!
MergePolicy mp = newTieredMergePolicy();
@@ -220,11 +216,6 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
Directory dir2 = newDirectory();
- if (dir2 instanceof MockDirectoryWrapper) {
- // Else, the virus checker may prevent deletion of files and cause
- // us to see too many bytes used by extension in the end:
- ((MockDirectoryWrapper) dir2).setEnableVirusScanner(false);
- }
mp = newTieredMergePolicy();
mp.setNoCFSRatio(0);
cfg = new IndexWriterConfig(new MockAnalyzer(random())).setUseCompoundFile(false).setMergePolicy(mp);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
index 2b88d74..9a58f77 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java
@@ -216,9 +216,6 @@ public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCas
try {
dir.setRandomIOExceptionRate(0.05);
dir.setRandomIOExceptionRateOnOpen(0.05);
- if (dir instanceof MockDirectoryWrapper) {
- dir.setEnableVirusScanner(false);
- }
verify(dir, docValues, null, numDims, numBytesPerDim, true);
} catch (IllegalStateException ise) {
if (ise.getMessage().contains("this writer hit an unrecoverable error")) {
@@ -901,13 +898,6 @@ public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCas
return x;
}
- private static Directory noVirusChecker(Directory dir) {
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
- }
- return dir;
- }
-
private Directory getDirectory(int numPoints) throws IOException {
Directory dir;
if (numPoints > 100000) {
@@ -915,7 +905,6 @@ public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCas
} else {
dir = newDirectory();
}
- noVirusChecker(dir);
//dir = FSDirectory.open(createTempDir());
return dir;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
index e6dd26e..914cd6d 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
@@ -145,7 +145,6 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
fieldsProducer = null;
dir.close();
- IOUtils.rm(path);
}
}
@@ -172,7 +171,6 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
fieldsProducer.close();
dir.close();
- IOUtils.rm(path);
}
protected static void checkReuse(TermsEnum termsEnum, int firstFlags, int secondFlags, boolean shouldReuse) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
index 27ba502..fe09389 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
@@ -1271,6 +1271,5 @@ public class RandomPostingsTester {
fieldsProducer.close();
dir.close();
- IOUtils.rm(path);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
index 3d6e43c..894086f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
@@ -654,7 +654,6 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas
TestUtil.checkIndex(dir);
dir.close();
- IOUtils.rm(tempDir);
if (VERBOSE) {
System.out.println("TEST: done [" + (System.currentTimeMillis()-t0) + " ms]");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
new file mode 100644
index 0000000..801a688
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
@@ -0,0 +1,65 @@
+package org.apache.lucene.mockfile;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Random;
+
+import org.apache.lucene.index.IndexWriter;
+
+/**
+ * Acts like Windows, where random programs may open the files you just wrote in an unfriendly
+ * way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc.
+ */
+public class VirusCheckingFS extends FilterFileSystemProvider {
+
+ // nocommit cannot use random here
+ final Random random;
+
+ private boolean enabled = true;
+
+ /**
+ * Create a new instance, wrapping {@code delegate}.
+ */
+ public VirusCheckingFS(FileSystem delegate, Random random) {
+ super("viruschecking://", delegate);
+ this.random = new Random(random.nextLong());
+ }
+
+ public void disable() {
+ enabled = false;
+ }
+
+ @Override
+ public void delete(Path path) throws IOException {
+
+ if (enabled // test infra disables when it's "really" time to delete after test is done
+ && Files.exists(path) // important that we NOT delay a NoSuchFileException until later
+ && path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
+ && random.nextInt(5) == 1) {
+ throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");
+ }
+ super.delete(path);
+ }
+
+ // TODO: rename? createOutput? deleteIfExists?
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
index c015351..db83153 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
@@ -164,7 +164,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
int count = dir.listAll().length;
dir.createOutput("foo.txt", IOContext.DEFAULT).close();
assertEquals(count+1, dir.listAll().length);
- dir.deleteFile("foo.txt");
+ dir.deleteFiles(Collections.singleton("foo.txt"));
assertEquals(count, dir.listAll().length);
dir.close();
}
@@ -668,6 +668,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
// LUCENE-3382 -- make sure we get exception if the directory really does not exist.
public void testNoDir() throws Throwable {
Path tempDir = createTempDir("doesnotexist");
+ assumeFalse("test directly deletes files", TestUtil.hasVirusChecker(tempDir));
IOUtils.rm(tempDir);
Directory dir = getDirectory(tempDir);
try {
@@ -750,9 +751,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
in2.close();
- dir.deleteFile("test");
- dir.deleteFile("test2");
-
+ dir.deleteFiles(Arrays.asList(new String[] {"test", "test2"}));
dir.close();
}
@@ -827,6 +826,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
// TODO: somehow change this test to
public void testFsyncDoesntCreateNewFiles() throws Exception {
Path path = createTempDir("nocreate");
+ assumeFalse("we directly delete files", TestUtil.hasVirusChecker(path));
Directory fsdir = getDirectory(path);
// this test backdoors the directory via the filesystem. so it must be an FSDir (for now)
@@ -1185,7 +1185,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
in.close();
}
Set<String> files = new HashSet<String>(Arrays.asList(dir.listAll()));
- // In case ExtraFS struck:
+ // In case ExtrasFS struck:
files.remove("extra0");
assertEquals(new HashSet<String>(names), files);
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
index 13bc0ff..5dfe3e8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
@@ -17,6 +17,17 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.Path;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -32,17 +43,7 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.PrintStreamInfoStream;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.nio.file.AccessDeniedException;
-import java.nio.file.Path;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
+import org.apache.lucene.util.TestUtil;
/** Base class for per-LockFactory tests. */
public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
@@ -54,7 +55,9 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
/** Test obtaining and releasing locks, checking validity */
public void testBasics() throws IOException {
- Directory dir = getDirectory(createTempDir());
+ Path tempPath = createTempDir();
+ assumeFalse("test works with lock files", TestUtil.hasVirusChecker(tempPath));
+ Directory dir = getDirectory(tempPath);
Lock l = dir.obtainLock("commit");
try {
@@ -72,7 +75,9 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
/** Test closing locks twice */
public void testDoubleClose() throws IOException {
- Directory dir = getDirectory(createTempDir());
+ Path tempPath = createTempDir();
+ assumeFalse("test works with lock files", TestUtil.hasVirusChecker(tempPath));
+ Directory dir = getDirectory(tempPath);
Lock l = dir.obtainLock("commit");
l.close();
@@ -83,18 +88,20 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
/** Test ensureValid returns true after acquire */
public void testValidAfterAcquire() throws IOException {
- Directory dir = getDirectory(createTempDir());
-
+ Path tempPath = createTempDir();
+ assumeFalse("test works with lock files", TestUtil.hasVirusChecker(tempPath));
+ Directory dir = getDirectory(tempPath);
Lock l = dir.obtainLock("commit");
l.ensureValid(); // no exception
l.close();
-
dir.close();
}
/** Test ensureValid throws exception after close */
public void testInvalidAfterClose() throws IOException {
- Directory dir = getDirectory(createTempDir());
+ Path tempPath = createTempDir();
+ assumeFalse("test works with lock files", TestUtil.hasVirusChecker(tempPath));
+ Directory dir = getDirectory(tempPath);
Lock l = dir.obtainLock("commit");
l.close();
@@ -103,12 +110,13 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
l.ensureValid();
fail("didn't get exception");
} catch (AlreadyClosedException expected) {}
-
dir.close();
}
public void testObtainConcurrently() throws InterruptedException, IOException {
- final Directory directory = getDirectory(createTempDir());
+ Path tempPath = createTempDir();
+ assumeFalse("test works with lock files", TestUtil.hasVirusChecker(tempPath));
+ final Directory directory = getDirectory(tempPath);
final AtomicBoolean running = new AtomicBoolean(true);
final AtomicInteger atomicCounter = new AtomicInteger(0);
final ReentrantLock assertingLock = new ReentrantLock();
@@ -156,7 +164,9 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
// IndexWriters over & over in 2 threads and making sure
// no unexpected exceptions are raised:
public void testStressLocks() throws Exception {
- Directory dir = getDirectory(createTempDir());
+ Path tempPath = createTempDir();
+ assumeFalse("test works with lock files", TestUtil.hasVirusChecker(tempPath));
+ Directory dir = getDirectory(tempPath);
// First create a 1 doc index:
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
index ec99c7e..3bd2319 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
@@ -73,12 +73,10 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
double randomIOExceptionRate;
double randomIOExceptionRateOnOpen;
Random randomState;
- boolean noDeleteOpenFile = true;
boolean assertNoDeleteOpenFile = false;
boolean preventDoubleWrite = true;
boolean trackDiskUsage = false;
boolean useSlowOpenClosers = LuceneTestCase.TEST_NIGHTLY;
- boolean enableVirusScanner = true;
boolean allowRandomFileNotFoundException = true;
boolean allowReadingFilesStillOpenForWrite = false;
private Set<String> unSyncedFiles;
@@ -105,15 +103,10 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
// is made to delete an open file, we enroll it here.
private Set<String> openFilesDeleted;
- // only tracked if virus scanner is enabled:
- // set of files it prevented deletion for
- private Set<String> triedToDelete;
-
private synchronized void init() {
if (openFiles == null) {
openFiles = new HashMap<>();
openFilesDeleted = new HashSet<>();
- triedToDelete = new HashSet<>();
}
if (createdFiles == null)
@@ -171,18 +164,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
allowReadingFilesStillOpenForWrite = value;
}
- /** Returns true if the virus scanner is enabled */
- public boolean getEnableVirusScanner() {
- return enableVirusScanner;
- }
-
- /** If set to true (the default), deleteFile sometimes
- * fails because a virus scanner is open.
- */
- public void setEnableVirusScanner(boolean value) {
- this.enableVirusScanner = value;
- }
-
/**
* Enum for controlling hard disk throttling.
* Set via {@link MockDirectoryWrapper #setThrottling(Throttling)}
@@ -237,12 +218,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
throw new IOException("cannot rename after crash");
}
- if (openFiles.containsKey(source)) {
- if (assertNoDeleteOpenFile) {
- throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: file \"" + source + "\" is still open: cannot rename"), source, true);
- } else if (noDeleteOpenFile) {
- throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + source + "\" is still open: cannot rename"), source, true);
- }
+ if (assertNoDeleteOpenFile && openFiles.containsKey(source)) {
+ throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: file \"" + source + "\" is still open: cannot rename"), source, true);
}
boolean success = false;
@@ -281,7 +258,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
/** Simulates a crash of OS or machine by overwriting
* unsynced files. */
public synchronized void crash() throws IOException {
- crashed = true;
openFiles = new HashMap<>();
openFilesForWrite = new HashSet<>();
openFilesDeleted = new HashSet<>();
@@ -303,7 +279,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
if (damage == 0) {
action = "deleted";
- deleteFile(name, true);
+ deleteFiles(Collections.singleton(name));
} else if (damage == 1) {
action = "zeroed";
// Zero out file entirely
@@ -337,28 +313,45 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
ii.close();
// Delete original and copy bytes back:
- deleteFile(name, true);
+ deleteFiles(Collections.singleton(name));
- final IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
- ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState));
- out.copyBytes(ii, ii.length());
- out.close();
- ii.close();
- deleteFile(tempFileName, true);
+ try(IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
+ ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState));
+ out.copyBytes(ii, ii.length());
+ ii.close();
+ } catch (IOException ioe) {
+ // VirusCheckingFS may have blocked the delete, at which point FSDir cannot overwrite here
+ if (ioe.getMessage().equals("file \"" + name + "\" is pending delete and cannot be overwritten")) {
+ // OK
+ action = "deleted";
+ } else {
+ throw ioe;
+ }
+ }
+ deleteFiles(Collections.singleton(tempFileName));
} else if (damage == 3) {
// The file survived intact:
action = "didn't change";
} else {
action = "fully truncated";
// Totally truncate the file to zero bytes
- deleteFile(name, true);
- IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
- out.close();
+ deleteFiles(Collections.singleton(name));
+ try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
+ } catch (IOException ioe) {
+ // VirusCheckingFS may have blocked the delete, at which point FSDir cannot overwrite here
+ if (ioe.getMessage().equals("file \"" + name + "\" is pending delete and cannot be overwritten")) {
+ // OK
+ action = "deleted";
+ } else {
+ throw ioe;
+ }
+ }
}
if (LuceneTestCase.VERBOSE) {
System.out.println("MockDirectoryWrapper: " + action + " unsynced file: " + name);
}
}
+ crashed = true;
}
public synchronized void clearCrash() {
@@ -385,18 +378,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
}
/**
- * Emulate windows whereby deleting an open file is not
- * allowed (raise IOException).
- */
- public void setNoDeleteOpenFile(boolean value) {
- this.noDeleteOpenFile = value;
- }
-
- public boolean getNoDeleteOpenFile() {
- return noDeleteOpenFile;
- }
-
- /**
* Trip a test assert if there is an attempt
* to delete an open file.
*/
@@ -468,9 +449,28 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
}
@Override
- public synchronized void deleteFile(String name) throws IOException {
+ public synchronized void deleteFiles(Collection<String> names) throws IOException {
maybeYield();
- deleteFile(name, false);
+
+ maybeThrowDeterministicException();
+
+ if (crashed) {
+ throw new IOException("cannot delete after crash");
+ }
+
+ for(String name : names) {
+ if (openFiles.containsKey(name)) {
+ openFilesDeleted.add(name);
+ if (assertNoDeleteOpenFile) {
+ throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
+ }
+ } else {
+ openFilesDeleted.remove(name);
+ }
+ }
+
+ unSyncedFiles.removeAll(names);
+ in.deleteFiles(names);
}
// sets the cause of the incoming ioe to be the stack
@@ -494,46 +494,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
}
}
- private synchronized void deleteFile(String name, boolean forced) throws IOException {
- maybeYield();
-
- maybeThrowDeterministicException();
-
- if (crashed && !forced)
- throw new IOException("cannot delete after crash");
-
- if (unSyncedFiles.contains(name))
- unSyncedFiles.remove(name);
- if (!forced && (noDeleteOpenFile || assertNoDeleteOpenFile)) {
- if (openFiles.containsKey(name)) {
- openFilesDeleted.add(name);
-
- if (!assertNoDeleteOpenFile) {
- throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
- } else {
- throw (AssertionError) fillOpenTrace(new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true);
- }
- } else {
- openFilesDeleted.remove(name);
- }
- }
- if (!forced && enableVirusScanner && (randomState.nextInt(4) == 0)) {
- triedToDelete.add(name);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MDW: now refuse to delete file: " + name);
- }
- throw new IOException("cannot delete file: " + name + ", a virus scanner has it open");
- }
- triedToDelete.remove(name);
- in.deleteFile(name);
- }
-
- /** Returns true if {@link #deleteFile} was called with this
- * fileName, but the virus checker prevented the deletion. */
- public boolean didTryToDelete(String fileName) {
- return triedToDelete.contains(fileName);
- }
-
public synchronized Set<String> getOpenDeletedFiles() {
return new HashSet<>(openFilesDeleted);
}
@@ -561,12 +521,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
throw new IOException("file \"" + name + "\" was already written to");
}
}
- if ((noDeleteOpenFile || assertNoDeleteOpenFile) && openFiles.containsKey(name)) {
- if (!assertNoDeleteOpenFile) {
- throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
- } else {
- throw new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
- }
+ if (assertNoDeleteOpenFile && openFiles.containsKey(name)) {
+ throw new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
}
unSyncedFiles.add(name);
@@ -751,9 +707,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
// files that we tried to delete, but couldn't because readers were open.
// all that matters is that we tried! (they will eventually go away)
// still open when we tried to delete
- Set<String> pendingDeletions = new HashSet<>(openFilesDeleted);
- // virus scanner when we tried to delete
- pendingDeletions.addAll(triedToDelete);
maybeYield();
if (openFiles == null) {
openFiles = new HashMap<>();
@@ -782,6 +735,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
if (getCheckIndexOnClose()) {
randomIOExceptionRate = 0.0;
randomIOExceptionRateOnOpen = 0.0;
+
if (DirectoryReader.indexExists(this)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now crush");
@@ -790,13 +744,14 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
+
+ // nocommit: we should also confirm all prior segments_N are not corrupt?
TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true);
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
if (assertNoUnreferencedFilesOnClose) {
// now look for unreferenced files: discount ones that we tried to delete but could not
Set<String> allFiles = new HashSet<>(Arrays.asList(listAll()));
- allFiles.removeAll(pendingDeletions);
String[] startFiles = allFiles.toArray(new String[0]);
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
@@ -806,51 +761,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
Set<String> startSet = new TreeSet<>(Arrays.asList(startFiles));
Set<String> endSet = new TreeSet<>(Arrays.asList(endFiles));
- if (pendingDeletions.contains("segments.gen") && endSet.contains("segments.gen")) {
- // this is possible if we hit an exception while writing segments.gen, we try to delete it
- // and it ends out in pendingDeletions (but IFD wont remove this).
- startSet.add("segments.gen");
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MDW: Unreferenced check: Ignoring segments.gen that we could not delete.");
- }
- }
-
- // it's possible we cannot delete the segments_N on windows if someone has it open and
- // maybe other files too, depending on timing. normally someone on windows wouldnt have
- // an issue (IFD would nuke this stuff eventually), but we pass NoDeletionPolicy...
- for (String file : pendingDeletions) {
- if (file.startsWith("segments") && !file.equals("segments.gen") && endSet.contains(file)) {
- startSet.add(file);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
- }
- SegmentInfos sis;
- try {
- sis = SegmentInfos.readCommit(in, file);
- } catch (IOException ioe) {
- // OK: likely some of the .si files were deleted
- sis = new SegmentInfos();
- }
-
- try {
- Set<String> ghosts = new HashSet<>(sis.files(false));
- for (String s : ghosts) {
- if (endSet.contains(s) && !startSet.contains(s)) {
- assert pendingDeletions.contains(s);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MDW: Unreferenced check: Ignoring referenced file: " + s + " " +
- "from " + file + " that we could not delete.");
- }
- startSet.add(s);
- }
- }
- } catch (Throwable t) {
- System.err.println("ERROR processing leftover segments file " + file + ":");
- t.printStackTrace();
- }
- }
- }
-
startFiles = startSet.toArray(new String[0]);
endFiles = endSet.toArray(new String[0]);
@@ -880,10 +790,6 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
}
- if (pendingDeletions.size() != 0) {
- extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
- }
-
throw new RuntimeException("unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
index 86f685a..27e4fb9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
@@ -39,6 +39,7 @@ import org.apache.lucene.mockfile.HandleLimitFS;
import org.apache.lucene.mockfile.LeakFS;
import org.apache.lucene.mockfile.ShuffleFS;
import org.apache.lucene.mockfile.VerboseFS;
+import org.apache.lucene.mockfile.VirusCheckingFS;
import org.apache.lucene.mockfile.WindowsFS;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase.SuppressFsync;
@@ -75,6 +76,11 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
private FileSystem fileSystem;
/**
+ * Only set if the file system chain includes the VirusCheckingFS
+ */
+ private VirusCheckingFS virusCheckingFS;
+
+ /**
* Suite failure marker.
*/
private final TestRuleMarkFailure failureMarker;
@@ -174,10 +180,17 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
if (allowed(avoid, ExtrasFS.class)) {
fs = new ExtrasFS(fs, random.nextInt(4) == 0, random.nextBoolean()).getFileSystem(null);
}
+ // nocommit true:
+ if (allowed(avoid, VirusCheckingFS.class) && (true || random.nextInt(10) == 1)) {
+ // 10% of the time we swap in virus checking (acts-like-windows) FS:
+ virusCheckingFS = new VirusCheckingFS(fs, random);
+ fs = virusCheckingFS.getFileSystem(null);
+ }
}
if (LuceneTestCase.VERBOSE) {
System.out.println("filesystem: " + fs.provider());
}
+
return fs.provider().getFileSystem(URI.create("file:///"));
}
@@ -211,6 +224,12 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
// was successful. Otherwise just report the path of temporary files
// and leave them there.
if (failureMarker.wasSuccessful()) {
+
+ if (virusCheckingFS != null) {
+ // Otherwise our IOUtils.rm below can fail:
+ virusCheckingFS.disable();
+ }
+
try {
IOUtils.rm(everything);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
index 99d4be3..89af2e2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
@@ -26,6 +26,7 @@ import java.io.PrintStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.CharBuffer;
+import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -90,10 +91,15 @@ import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.mockfile.FilterFileSystem;
+import org.apache.lucene.mockfile.VirusCheckingFS;
+import org.apache.lucene.mockfile.WindowsFS;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RAMDirectory;
@@ -102,6 +108,7 @@ import org.junit.Assert;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
/**
* General utility methods for Lucene unit tests.
*/
@@ -1281,4 +1288,43 @@ public final class TestUtil {
}
return ram;
}
+
+ public static boolean hasWindowsFS(Directory dir) {
+ dir = FilterDirectory.unwrap(dir);
+ if (dir instanceof FSDirectory) {
+ Path path = ((FSDirectory) dir).getDirectory();
+ FileSystem fs = path.getFileSystem();
+ while (fs instanceof FilterFileSystem) {
+ FilterFileSystem ffs = (FilterFileSystem) fs;
+ if (ffs.getParent() instanceof WindowsFS) {
+ return true;
+ }
+ fs = ffs.getDelegate();
+ }
+ }
+
+ return false;
+ }
+
+ public static boolean hasVirusChecker(Directory dir) {
+ dir = FilterDirectory.unwrap(dir);
+ if (dir instanceof FSDirectory) {
+ return hasVirusChecker(((FSDirectory) dir).getDirectory());
+ } else {
+ return false;
+ }
+ }
+
+ public static boolean hasVirusChecker(Path path) {
+ FileSystem fs = path.getFileSystem();
+ while (fs instanceof FilterFileSystem) {
+ FilterFileSystem ffs = (FilterFileSystem) fs;
+ if (ffs.getParent() instanceof VirusCheckingFS) {
+ return true;
+ }
+ fs = ffs.getDelegate();
+ }
+
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
index a2ace61..3800ff7 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
@@ -317,7 +317,7 @@ public class FSTTester<T> {
fst = new FST<>(in, outputs);
} finally {
in.close();
- dir.deleteFile("fst.bin");
+ dir.deleteFiles(Collections.singleton("fst.bin"));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
index d416f8a..f69ea19 100644
--- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
+++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
@@ -55,10 +55,6 @@ public class TestCompressingStoredFieldsFormat extends BaseStoredFieldsFormatTes
public void testDeletePartiallyWrittenFilesIfAbort() throws IOException {
Directory dir = newDirectory();
- // test explicitly needs files to always be actually deleted
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
- }
IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
iwConf.setCodec(CompressingCodec.randomInstance(random()));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestVirusCheckingFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestVirusCheckingFS.java b/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestVirusCheckingFS.java
new file mode 100644
index 0000000..17b1575
--- /dev/null
+++ b/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestVirusCheckingFS.java
@@ -0,0 +1,62 @@
+package org.apache.lucene.mockfile;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Basic tests for VirusCheckingFS */
+public class TestVirusCheckingFS extends MockFileSystemTestCase {
+
+ @Override
+ protected Path wrap(Path path) {
+ FileSystem fs = new VirusCheckingFS(path.getFileSystem(), random()).getFileSystem(URI.create("file:///"));
+ return new FilterPath(path, fs);
+ }
+
+ /** Test Files.delete fails if a file has an open inputstream against it */
+ public void testDeleteSometimesFails() throws IOException {
+ Path dir = wrap(createTempDir());
+
+ int counter = 0;
+ while (true) {
+ Path path = dir.resolve("file" + counter);
+ counter++;
+
+ OutputStream file = Files.newOutputStream(path);
+ file.write(5);
+ file.close();
+
+ // File is now closed, we attempt delete:
+ try {
+ Files.delete(path);
+ } catch (AccessDeniedException ade) {
+ // expected (sometimes)
+ assertTrue(ade.getMessage().contains("VirusCheckingFS is randomly refusing to delete file "));
+ break;
+ }
+
+ assertFalse(Files.exists(path));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestWindowsFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestWindowsFS.java b/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestWindowsFS.java
index 333ff00..7b80604 100644
--- a/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestWindowsFS.java
+++ b/lucene/test-framework/src/test/org/apache/lucene/mockfile/TestWindowsFS.java
@@ -41,10 +41,10 @@ import org.apache.lucene.util.Constants;
/** Basic tests for WindowsFS */
public class TestWindowsFS extends MockFileSystemTestCase {
- // currently we don't emulate windows well enough to work on windows!
@Override
public void setUp() throws Exception {
super.setUp();
+ // irony: currently we don't emulate windows well enough to work on windows!
assumeFalse("windows is not supported", Constants.WINDOWS);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
index fd30582..92bf568 100644
--- a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
+++ b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
@@ -37,7 +37,6 @@ public class TestMockDirectoryWrapper extends BaseDirectoryTestCase {
} else {
dir = newMockFSDirectory(path);
}
- dir.setEnableVirusScanner(false); // test manipulates filesystem directly
return dir;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index 715b6a6..cf7220d 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -155,7 +156,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
*/
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
toDir.copyFrom(fromDir, fileName, fileName, ioContext);
- fromDir.deleteFile(fileName);
+ fromDir.deleteFiles(Collections.singleton(fileName));
}
/**
@@ -248,9 +249,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
try {
contents = dir.listAll();
if (contents != null) {
- for (String file : contents) {
- dir.deleteFile(file);
- }
+ dir.deleteFiles(Arrays.asList(contents));
}
} catch (IOException e) {
SolrException.log(log, "Error deleting files from Directory", e);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 862d4e7..86aaf95 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -1206,7 +1206,7 @@ public class IndexFetcher {
}
}
try {
- dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
+ dir.deleteFiles(Collections.singleton(IndexFetcher.INDEX_PROPERTIES));
} catch (IOException e) {
// no problem
}
@@ -1651,7 +1651,7 @@ public class IndexFetcher {
}
public void delete() throws Exception {
- copy2Dir.deleteFile(saveAs);
+ copy2Dir.deleteFiles(Collections.singleton(saveAs));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84f44589/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
index d6a7e1d..cf426ce 100644
--- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
+++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Collections;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
@@ -111,7 +112,7 @@ public class RestoreCore implements Callable<Boolean> {
try {
dir = core.getDirectoryFactory().get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA,
core.getSolrConfig().indexConfig.lockType);
- dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
+ dir.deleteFiles(Collections.singleton(IndexFetcher.INDEX_PROPERTIES));
} finally {
if (dir != null) {
core.getDirectoryFactory().release(dir);