You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/06/20 13:34:10 UTC
lucene-solr:branch_6x: LUCENE-7345: make sure MDW's efforts to
corrumpt un-sync'd files is successful
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x bf10bfcaa -> 0a0865b2c
LUCENE-7345: make sure MDW's efforts to corrumpt un-sync'd files is successful
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0a0865b2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0a0865b2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0a0865b2
Branch: refs/heads/branch_6x
Commit: 0a0865b2c671e2951bde0bc07a2f4ef73314ed4f
Parents: bf10bfc
Author: Mike McCandless <mi...@apache.org>
Authored: Mon Jun 20 09:17:44 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Mon Jun 20 09:18:13 2016 -0400
----------------------------------------------------------------------
.../lucene/store/MockDirectoryWrapper.java | 81 ++++++++++---------
.../lucene/store/TestMockDirectoryWrapper.java | 82 +++++++++++++++++++-
2 files changed, 126 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0a0865b2/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
index 307a464..209ee80 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
@@ -88,6 +88,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
private ThrottledIndexOutput throttledOutput;
private Throttling throttling = LuceneTestCase.TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER;
+ // for testing
+ boolean alwaysCorrupt;
+
final AtomicInteger inputCloneCount = new AtomicInteger();
// use this for tracking files for crash.
@@ -297,6 +300,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
CollectionUtil.timSort(filesToCorrupt);
for(String name : filesToCorrupt) {
int damage = randomState.nextInt(6);
+ if (alwaysCorrupt && damage == 3) {
+ damage = 4;
+ }
String action = null;
switch(damage) {
@@ -313,9 +319,12 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
try {
length = fileLength(name);
} catch (IOException ioe) {
- // Ignore
- continue;
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
+
+ // Delete original and write zeros back:
+ deleteFile(name);
+
byte[] zeroes = new byte[256];
long upto = 0;
try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
@@ -325,7 +334,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
upto += limit;
}
} catch (IOException ioe) {
- // ignore
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
break;
@@ -341,8 +350,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
tempFileName = tempOut.getName();
tempOut.copyBytes(ii, ii.length()/2);
- } catch (IOException ioe) {
- // ignore
+ } catch (IOException ioe) {
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
// Delete original and copy bytes back:
@@ -351,8 +360,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
out.copyBytes(ii, ii.length());
- } catch (IOException ioe) {
- // ignore
+ } catch (IOException ioe) {
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
deleteFile(tempFileName);
}
@@ -371,32 +380,32 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
String tempFileName = null;
try (IndexOutput tempOut = in.createTempOutput("name", "mdw_corrupt", LuceneTestCase.newIOContext(randomState));
IndexInput ii = in.openInput(name, LuceneTestCase.newIOContext(randomState))) {
- tempFileName = tempOut.getName();
- if (ii.length() > 0) {
- // Copy first part unchanged:
- long byteToCorrupt = (long) (randomState.nextDouble() * ii.length());
- if (byteToCorrupt > 0) {
- tempOut.copyBytes(ii, byteToCorrupt);
- }
-
- // Randomly flip one bit from this byte:
- byte b = ii.readByte();
- int bitToFlip = randomState.nextInt(8);
- b = (byte) (b ^ (1 << bitToFlip));
- tempOut.writeByte(b);
-
- action = "flip bit " + bitToFlip + " of byte " + byteToCorrupt + " out of " + ii.length() + " bytes";
-
- // Copy last part unchanged:
- long bytesLeft = ii.length() - byteToCorrupt - 1;
- if (bytesLeft > 0) {
- tempOut.copyBytes(ii, bytesLeft);
- }
- } else {
- action = "didn't change";
+ tempFileName = tempOut.getName();
+ if (ii.length() > 0) {
+ // Copy first part unchanged:
+ long byteToCorrupt = (long) (randomState.nextDouble() * ii.length());
+ if (byteToCorrupt > 0) {
+ tempOut.copyBytes(ii, byteToCorrupt);
}
- } catch (IOException ioe) {
- // ignore
+
+ // Randomly flip one bit from this byte:
+ byte b = ii.readByte();
+ int bitToFlip = randomState.nextInt(8);
+ b = (byte) (b ^ (1 << bitToFlip));
+ tempOut.writeByte(b);
+
+ action = "flip bit " + bitToFlip + " of byte " + byteToCorrupt + " out of " + ii.length() + " bytes";
+
+ // Copy last part unchanged:
+ long bytesLeft = ii.length() - byteToCorrupt - 1;
+ if (bytesLeft > 0) {
+ tempOut.copyBytes(ii, bytesLeft);
+ }
+ } else {
+ action = "didn't change";
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
// Delete original and copy bytes back:
@@ -404,9 +413,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
IndexInput ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState))) {
- out.copyBytes(ii, ii.length());
- } catch (IOException ioe) {
- // ignore
+ out.copyBytes(ii, ii.length());
+ } catch (IOException ioe) {
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
deleteFile(tempFileName);
@@ -421,7 +430,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
try (IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState))) {
out.getFilePointer(); // just fake access to prevent compiler warning
} catch (IOException ioe) {
- // ignore
+ throw new RuntimeException("hit unexpected IOException while trying to corrupt file " + name, ioe);
}
break;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0a0865b2/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
index ef7b0f3..ae453e5 100644
--- a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
+++ b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
@@ -16,7 +16,10 @@
*/
package org.apache.lucene.store;
+import java.io.EOFException;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import org.apache.lucene.document.Document;
@@ -90,5 +93,82 @@ public class TestMockDirectoryWrapper extends BaseDirectoryTestCase {
iw.commit();
iw.close();
dir.close();
- }
+ }
+
+ // just shields the wrapped directory from being closed
+ private static class PreventCloseDirectoryWrapper extends FilterDirectory {
+ public PreventCloseDirectoryWrapper(Directory in) {
+ super(in);
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
+ public void testCorruptOnCloseIsWorkingFSDir() throws Exception {
+ Path path = createTempDir();
+ try(Directory dir = newFSDirectory(path)) {
+ testCorruptOnCloseIsWorking(dir);
+ }
+ }
+
+ public void testCorruptOnCloseIsWorkingRAMDir() throws Exception {
+ try(Directory dir = new RAMDirectory()) {
+ testCorruptOnCloseIsWorking(dir);
+ }
+ }
+
+ private void testCorruptOnCloseIsWorking(Directory dir) throws Exception {
+
+ dir = new PreventCloseDirectoryWrapper(dir);
+
+ try (MockDirectoryWrapper wrapped = new MockDirectoryWrapper(random(), dir)) {
+
+ // otherwise MDW sometimes randomly leaves the file intact and we'll see false test failures:
+ wrapped.alwaysCorrupt = true;
+
+ // MDW will only try to corrupt things if it sees an index:
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ iw.addDocument(new Document());
+ iw.close();
+
+ // not sync'd!
+ try (IndexOutput out = wrapped.createOutput("foo", IOContext.DEFAULT)) {
+ for(int i=0;i<100;i++) {
+ out.writeInt(i);
+ }
+ }
+
+ // MDW.close now corrupts our unsync'd file (foo):
+ }
+
+ boolean changed = false;
+ IndexInput in = null;
+ try {
+ in = dir.openInput("foo", IOContext.DEFAULT);
+ } catch (NoSuchFileException | FileNotFoundException fnfe) {
+ // ok
+ changed = true;
+ }
+ if (in != null) {
+ for(int i=0;i<100;i++) {
+ int x;
+ try {
+ x = in.readInt();
+ } catch (EOFException eofe) {
+ changed = true;
+ break;
+ }
+ if (x != i) {
+ changed = true;
+ break;
+ }
+ }
+
+ in.close();
+ }
+
+ assertTrue("MockDirectoryWrapper on dir=" + dir + " failed to corrupt an unsync'd file", changed);
+ }
}