You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2018/07/24 06:55:09 UTC
[1/2] lucene-solr:master: LUCENE-8415: Clean up Directory contracts
(write-once,
no reads-before-write-completed). Minor test improvements and cleanups.
Repository: lucene-solr
Updated Branches:
refs/heads/master 9d7aef6e7 -> 8892c0d9a
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
index bb9b711..4270424 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
@@ -19,6 +19,9 @@ package org.apache.lucene.store;
import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
@@ -29,201 +32,195 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.Set;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import java.util.zip.CRC32;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.generators.RandomBytes;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexNotFoundException;
+import org.apache.lucene.mockfile.ExtrasFS;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import org.junit.Assert;
-/** Base class for per-Directory tests. */
-
+/**
+ * Base class for per-Directory tests.
+ */
public abstract class BaseDirectoryTestCase extends LuceneTestCase {
- /** Subclass returns the Directory to be tested; if it's
+ /** A subclass returns the Directory to be tested; if it's
* an FS-based directory it should point to the specified
* path, else it can ignore it. */
protected abstract Directory getDirectory(Path path) throws IOException;
-
- // first some basic tests for the directory api
-
+
public void testCopyFrom() throws Exception {
- Directory source = getDirectory(createTempDir("testCopy"));
- Directory dest = newDirectory();
-
- IndexOutput output = source.createOutput("foobar", newIOContext(random()));
- int numBytes = random().nextInt(20000);
- byte bytes[] = new byte[numBytes];
- random().nextBytes(bytes);
- output.writeBytes(bytes, bytes.length);
- output.close();
-
- dest.copyFrom(source, "foobar", "foobaz", newIOContext(random()));
- assertTrue(slowFileExists(dest, "foobaz"));
-
- IndexInput input = dest.openInput("foobaz", newIOContext(random()));
- byte bytes2[] = new byte[numBytes];
- input.readBytes(bytes2, 0, bytes2.length);
- assertEquals(input.length(), numBytes);
- input.close();
-
- assertArrayEquals(bytes, bytes2);
-
- IOUtils.close(source, dest);
+ try (Directory source = getDirectory(createTempDir("testCopy"));
+ Directory dest = newDirectory()) {
+ runCopyFrom(source, dest);
+ }
+
+ try (Directory source = newDirectory();
+ Directory dest = getDirectory(createTempDir("testCopyDestination"))) {
+ runCopyFrom(source, dest);
+ }
}
-
- public void testCopyFromDestination() throws Exception {
- Directory source = newDirectory();
- Directory dest = getDirectory(createTempDir("testCopyDestination"));
-
+
+ private void runCopyFrom(Directory source, Directory dest) throws IOException {
IndexOutput output = source.createOutput("foobar", newIOContext(random()));
- int numBytes = random().nextInt(20000);
- byte bytes[] = new byte[numBytes];
- random().nextBytes(bytes);
+
+ byte bytes[] = RandomBytes.randomBytesOfLength(random(), 20000);
output.writeBytes(bytes, bytes.length);
output.close();
-
+
dest.copyFrom(source, "foobar", "foobaz", newIOContext(random()));
assertTrue(slowFileExists(dest, "foobaz"));
-
+
IndexInput input = dest.openInput("foobaz", newIOContext(random()));
- byte bytes2[] = new byte[numBytes];
+ byte bytes2[] = new byte[bytes.length];
input.readBytes(bytes2, 0, bytes2.length);
- assertEquals(input.length(), numBytes);
input.close();
-
+
assertArrayEquals(bytes, bytes2);
-
- IOUtils.close(source, dest);
}
-
+
public void testRename() throws Exception {
- Directory dir = getDirectory(createTempDir("testRename"));
-
- IndexOutput output = dir.createOutput("foobar", newIOContext(random()));
- int numBytes = random().nextInt(20000);
- byte bytes[] = new byte[numBytes];
- random().nextBytes(bytes);
- output.writeBytes(bytes, bytes.length);
- output.close();
-
- dir.rename("foobar", "foobaz");
-
- IndexInput input = dir.openInput("foobaz", newIOContext(random()));
- byte bytes2[] = new byte[numBytes];
- input.readBytes(bytes2, 0, bytes2.length);
- assertEquals(input.length(), numBytes);
- input.close();
-
- assertArrayEquals(bytes, bytes2);
-
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testRename"))) {
+ IndexOutput output = dir.createOutput("foobar", newIOContext(random()));
+ int numBytes = random().nextInt(20000);
+ byte bytes[] = new byte[numBytes];
+ random().nextBytes(bytes);
+ output.writeBytes(bytes, bytes.length);
+ output.close();
+
+ dir.rename("foobar", "foobaz");
+
+ IndexInput input = dir.openInput("foobaz", newIOContext(random()));
+ byte bytes2[] = new byte[numBytes];
+ input.readBytes(bytes2, 0, bytes2.length);
+ assertEquals(input.length(), numBytes);
+ input.close();
+
+ assertArrayEquals(bytes, bytes2);
+ }
}
-
+
public void testDeleteFile() throws Exception {
- Directory dir = getDirectory(createTempDir("testDeleteFile"));
- int count = dir.listAll().length;
- dir.createOutput("foo.txt", IOContext.DEFAULT).close();
- assertEquals(count+1, dir.listAll().length);
- dir.deleteFile("foo.txt");
- assertEquals(count, dir.listAll().length);
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testDeleteFile"))) {
+ String file = "foo.txt";
+ Assert.assertFalse(Arrays.asList(dir.listAll()).contains(file));
+
+ dir.createOutput("foo.txt", IOContext.DEFAULT).close();
+ Assert.assertTrue(Arrays.asList(dir.listAll()).contains(file));
+
+ dir.deleteFile("foo.txt");
+ Assert.assertFalse(Arrays.asList(dir.listAll()).contains(file));
+
+ expectThrowsAnyOf(Arrays.asList(NoSuchFileException.class, FileNotFoundException.class), () -> {
+ dir.deleteFile("foo.txt");
+ });
+ }
}
-
+
public void testByte() throws Exception {
- Directory dir = getDirectory(createTempDir("testByte"));
- IndexOutput output = dir.createOutput("byte", newIOContext(random()));
- output.writeByte((byte)128);
- output.close();
-
- IndexInput input = dir.openInput("byte", newIOContext(random()));
- assertEquals(1, input.length());
- assertEquals((byte)128, input.readByte());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testByte"))) {
+ IndexOutput output = dir.createOutput("byte", newIOContext(random()));
+ output.writeByte((byte) 128);
+ output.close();
+
+ IndexInput input = dir.openInput("byte", newIOContext(random()));
+ assertEquals(1, input.length());
+ assertEquals((byte) 128, input.readByte());
+ input.close();
+ }
}
-
+
public void testShort() throws Exception {
- Directory dir = getDirectory(createTempDir("testShort"));
- IndexOutput output = dir.createOutput("short", newIOContext(random()));
- output.writeShort((short)-20);
- output.close();
-
- IndexInput input = dir.openInput("short", newIOContext(random()));
- assertEquals(2, input.length());
- assertEquals((short)-20, input.readShort());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testShort"))) {
+ IndexOutput output = dir.createOutput("short", newIOContext(random()));
+ output.writeShort((short) -20);
+ output.close();
+
+ IndexInput input = dir.openInput("short", newIOContext(random()));
+ assertEquals(2, input.length());
+ assertEquals((short) -20, input.readShort());
+ input.close();
+ }
}
-
+
public void testInt() throws Exception {
- Directory dir = getDirectory(createTempDir("testInt"));
- IndexOutput output = dir.createOutput("int", newIOContext(random()));
- output.writeInt(-500);
- output.close();
-
- IndexInput input = dir.openInput("int", newIOContext(random()));
- assertEquals(4, input.length());
- assertEquals(-500, input.readInt());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testInt"))) {
+ IndexOutput output = dir.createOutput("int", newIOContext(random()));
+ output.writeInt(-500);
+ output.close();
+
+ IndexInput input = dir.openInput("int", newIOContext(random()));
+ assertEquals(4, input.length());
+ assertEquals(-500, input.readInt());
+ input.close();
+ }
}
-
+
public void testLong() throws Exception {
- Directory dir = getDirectory(createTempDir("testLong"));
- IndexOutput output = dir.createOutput("long", newIOContext(random()));
- output.writeLong(-5000);
- output.close();
-
- IndexInput input = dir.openInput("long", newIOContext(random()));
- assertEquals(8, input.length());
- assertEquals(-5000L, input.readLong());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testLong"))) {
+ IndexOutput output = dir.createOutput("long", newIOContext(random()));
+ output.writeLong(-5000);
+ output.close();
+
+ IndexInput input = dir.openInput("long", newIOContext(random()));
+ assertEquals(8, input.length());
+ assertEquals(-5000L, input.readLong());
+ input.close();
+ }
}
-
+
public void testString() throws Exception {
- Directory dir = getDirectory(createTempDir("testString"));
- IndexOutput output = dir.createOutput("string", newIOContext(random()));
- output.writeString("hello!");
- output.close();
-
- IndexInput input = dir.openInput("string", newIOContext(random()));
- assertEquals("hello!", input.readString());
- assertEquals(7, input.length());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testString"))) {
+ IndexOutput output = dir.createOutput("string", newIOContext(random()));
+ output.writeString("hello!");
+ output.close();
+
+ IndexInput input = dir.openInput("string", newIOContext(random()));
+ assertEquals("hello!", input.readString());
+ assertEquals(7, input.length());
+ input.close();
+ }
}
-
+
public void testVInt() throws Exception {
- Directory dir = getDirectory(createTempDir("testVInt"));
- IndexOutput output = dir.createOutput("vint", newIOContext(random()));
- output.writeVInt(500);
- output.close();
-
- IndexInput input = dir.openInput("vint", newIOContext(random()));
- assertEquals(2, input.length());
- assertEquals(500, input.readVInt());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testVInt"))) {
+ IndexOutput output = dir.createOutput("vint", newIOContext(random()));
+ output.writeVInt(500);
+ output.close();
+
+ IndexInput input = dir.openInput("vint", newIOContext(random()));
+ assertEquals(2, input.length());
+ assertEquals(500, input.readVInt());
+ input.close();
+ }
}
-
+
public void testVLong() throws Exception {
- Directory dir = getDirectory(createTempDir("testVLong"));
- IndexOutput output = dir.createOutput("vlong", newIOContext(random()));
- output.writeVLong(Long.MAX_VALUE);
- output.close();
-
- IndexInput input = dir.openInput("vlong", newIOContext(random()));
- assertEquals(9, input.length());
- assertEquals(Long.MAX_VALUE, input.readVLong());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testVLong"))) {
+ IndexOutput output = dir.createOutput("vlong", newIOContext(random()));
+ output.writeVLong(Long.MAX_VALUE);
+ output.close();
+
+ IndexInput input = dir.openInput("vlong", newIOContext(random()));
+ assertEquals(9, input.length());
+ assertEquals(Long.MAX_VALUE, input.readVLong());
+ input.close();
+ }
}
-
+
public void testZInt() throws Exception {
final int[] ints = new int[random().nextInt(10)];
for (int i = 0; i < ints.length; ++i) {
@@ -241,22 +238,23 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
throw new AssertionError();
}
}
- Directory dir = getDirectory(createTempDir("testZInt"));
- IndexOutput output = dir.createOutput("zint", newIOContext(random()));
- for (int i : ints) {
- output.writeZInt(i);
- }
- output.close();
-
- IndexInput input = dir.openInput("zint", newIOContext(random()));
- for (int i : ints) {
- assertEquals(i, input.readZInt());
+
+ try (Directory dir = getDirectory(createTempDir("testZInt"))) {
+ IndexOutput output = dir.createOutput("zint", newIOContext(random()));
+ for (int i : ints) {
+ output.writeZInt(i);
+ }
+ output.close();
+
+ IndexInput input = dir.openInput("zint", newIOContext(random()));
+ for (int i : ints) {
+ assertEquals(i, input.readZInt());
+ }
+ assertEquals(input.length(), input.getFilePointer());
+ input.close();
}
- assertEquals(input.length(), input.getFilePointer());
- input.close();
- dir.close();
}
-
+
public void testZLong() throws Exception {
final long[] longs = new long[random().nextInt(10)];
for (int i = 0; i < longs.length; ++i) {
@@ -274,97 +272,98 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
throw new AssertionError();
}
}
- Directory dir = getDirectory(createTempDir("testZLong"));
- IndexOutput output = dir.createOutput("zlong", newIOContext(random()));
- for (long l : longs) {
- output.writeZLong(l);
- }
- output.close();
-
- IndexInput input = dir.openInput("zlong", newIOContext(random()));
- for (long l : longs) {
- assertEquals(l, input.readZLong());
+
+ try (Directory dir = getDirectory(createTempDir("testZLong"))) {
+ IndexOutput output = dir.createOutput("zlong", newIOContext(random()));
+ for (long l : longs) {
+ output.writeZLong(l);
+ }
+ output.close();
+
+ IndexInput input = dir.openInput("zlong", newIOContext(random()));
+ for (long l : longs) {
+ assertEquals(l, input.readZLong());
+ }
+ assertEquals(input.length(), input.getFilePointer());
+ input.close();
}
- assertEquals(input.length(), input.getFilePointer());
- input.close();
- dir.close();
}
public void testSetOfStrings() throws Exception {
- Directory dir = getDirectory(createTempDir("testSetOfStrings"));
-
- IndexOutput output = dir.createOutput("stringset", newIOContext(random()));
- output.writeSetOfStrings(asSet("test1", "test2"));
- output.writeSetOfStrings(Collections.emptySet());
- output.writeSetOfStrings(asSet("test3"));
- output.close();
-
- IndexInput input = dir.openInput("stringset", newIOContext(random()));
- Set<String> set1 = input.readSetOfStrings();
- assertEquals(asSet("test1", "test2"), set1);
- // set should be immutable
- expectThrows(UnsupportedOperationException.class, () -> {
- set1.add("bogus");
- });
-
- Set<String> set2 = input.readSetOfStrings();
- assertEquals(Collections.emptySet(), set2);
- // set should be immutable
- expectThrows(UnsupportedOperationException.class, () -> {
- set2.add("bogus");
- });
-
- Set<String> set3 = input.readSetOfStrings();
- assertEquals(Collections.singleton("test3"), set3);
- // set should be immutable
- expectThrows(UnsupportedOperationException.class, () -> {
- set3.add("bogus");
- });
-
- assertEquals(input.length(), input.getFilePointer());
- input.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testSetOfStrings"))) {
+
+ IndexOutput output = dir.createOutput("stringset", newIOContext(random()));
+ output.writeSetOfStrings(asSet("test1", "test2"));
+ output.writeSetOfStrings(Collections.emptySet());
+ output.writeSetOfStrings(asSet("test3"));
+ output.close();
+
+ IndexInput input = dir.openInput("stringset", newIOContext(random()));
+ Set<String> set1 = input.readSetOfStrings();
+ assertEquals(asSet("test1", "test2"), set1);
+ // set should be immutable
+ expectThrows(UnsupportedOperationException.class, () -> {
+ set1.add("bogus");
+ });
+
+ Set<String> set2 = input.readSetOfStrings();
+ assertEquals(Collections.emptySet(), set2);
+ // set should be immutable
+ expectThrows(UnsupportedOperationException.class, () -> {
+ set2.add("bogus");
+ });
+
+ Set<String> set3 = input.readSetOfStrings();
+ assertEquals(Collections.singleton("test3"), set3);
+ // set should be immutable
+ expectThrows(UnsupportedOperationException.class, () -> {
+ set3.add("bogus");
+ });
+
+ assertEquals(input.length(), input.getFilePointer());
+ input.close();
+ }
}
-
+
public void testMapOfStrings() throws Exception {
Map<String,String> m = new HashMap<>();
m.put("test1", "value1");
m.put("test2", "value2");
-
- Directory dir = getDirectory(createTempDir("testMapOfStrings"));
- IndexOutput output = dir.createOutput("stringmap", newIOContext(random()));
- output.writeMapOfStrings(m);
- output.writeMapOfStrings(Collections.emptyMap());
- output.writeMapOfStrings(Collections.singletonMap("key", "value"));
- output.close();
-
- IndexInput input = dir.openInput("stringmap", newIOContext(random()));
- Map<String,String> map1 = input.readMapOfStrings();
- assertEquals(m, map1);
- // map should be immutable
- expectThrows(UnsupportedOperationException.class, () -> {
- map1.put("bogus1", "bogus2");
- });
-
- Map<String,String> map2 = input.readMapOfStrings();
- assertEquals(Collections.emptyMap(), map2);
- // map should be immutable
- expectThrows(UnsupportedOperationException.class, () -> {
- map2.put("bogus1", "bogus2");
- });
-
- Map<String,String> map3 = input.readMapOfStrings();
- assertEquals(Collections.singletonMap("key", "value"), map3);
- // map should be immutable
- expectThrows(UnsupportedOperationException.class, () -> {
- map3.put("bogus1", "bogus2");
- });
-
- assertEquals(input.length(), input.getFilePointer());
- input.close();
- dir.close();
+
+ try (Directory dir = getDirectory(createTempDir("testMapOfStrings"))) {
+ IndexOutput output = dir.createOutput("stringmap", newIOContext(random()));
+ output.writeMapOfStrings(m);
+ output.writeMapOfStrings(Collections.emptyMap());
+ output.writeMapOfStrings(Collections.singletonMap("key", "value"));
+ output.close();
+
+ IndexInput input = dir.openInput("stringmap", newIOContext(random()));
+ Map<String, String> map1 = input.readMapOfStrings();
+ assertEquals(m, map1);
+ // map should be immutable
+ expectThrows(UnsupportedOperationException.class, () -> {
+ map1.put("bogus1", "bogus2");
+ });
+
+ Map<String, String> map2 = input.readMapOfStrings();
+ assertEquals(Collections.emptyMap(), map2);
+ // map should be immutable
+ expectThrows(UnsupportedOperationException.class, () -> {
+ map2.put("bogus1", "bogus2");
+ });
+
+ Map<String, String> map3 = input.readMapOfStrings();
+ assertEquals(Collections.singletonMap("key", "value"), map3);
+ // map should be immutable
+ expectThrows(UnsupportedOperationException.class, () -> {
+ map3.put("bogus1", "bogus2");
+ });
+
+ assertEquals(input.length(), input.getFilePointer());
+ input.close();
+ }
}
-
+
// TODO: fold in some of the testing of o.a.l.index.TestIndexInput in here!
public void testChecksum() throws Exception {
CRC32 expected = new CRC32();
@@ -372,650 +371,594 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
byte bytes[] = new byte[numBytes];
random().nextBytes(bytes);
expected.update(bytes);
-
- Directory dir = getDirectory(createTempDir("testChecksum"));
- IndexOutput output = dir.createOutput("checksum", newIOContext(random()));
- output.writeBytes(bytes, 0, bytes.length);
- output.close();
-
- ChecksumIndexInput input = dir.openChecksumInput("checksum", newIOContext(random()));
- input.skipBytes(numBytes);
-
- assertEquals(expected.getValue(), input.getChecksum());
- input.close();
- dir.close();
+
+ try (Directory dir = getDirectory(createTempDir("testChecksum"))) {
+ IndexOutput output = dir.createOutput("checksum", newIOContext(random()));
+ output.writeBytes(bytes, 0, bytes.length);
+ output.close();
+
+ ChecksumIndexInput input = dir.openChecksumInput("checksum", newIOContext(random()));
+ input.skipBytes(numBytes);
+
+ assertEquals(expected.getValue(), input.getChecksum());
+ input.close();
+ }
}
-
+
/** Make sure directory throws AlreadyClosedException if
* you try to createOutput after closing. */
public void testDetectClose() throws Throwable {
Directory dir = getDirectory(createTempDir("testDetectClose"));
dir.close();
- expectThrows(AlreadyClosedException.class, () -> {
+
+ expectThrows(AlreadyClosedException.class, () -> {
dir.createOutput("test", newIOContext(random()));
});
}
-
- public void testThreadSafety() throws Exception {
- final Directory dir = getDirectory(createTempDir("testThreadSafety"));
- if (dir instanceof BaseDirectoryWrapper) {
- ((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we arent making an index
- }
- if (dir instanceof MockDirectoryWrapper) {
- ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); // makes this test really slow
- }
-
- if (VERBOSE) {
- System.out.println(dir);
- }
-
- class TheThread extends Thread {
- private String name;
- public TheThread(String name) {
- this.name = name;
+ public void testThreadSafetyInListAll() throws Exception {
+ try (Directory dir = getDirectory(createTempDir("testThreadSafety"))) {
+ if (dir instanceof BaseDirectoryWrapper) {
+ // we are not making a real index, just writing, reading files.
+ ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false);
+ }
+ if (dir instanceof MockDirectoryWrapper) {
+ // makes this test really slow
+ ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
}
-
- @Override
- public void run() {
- for (int i = 0; i < 1000; i++) {
- String fileName = this.name + i;
- try {
- //System.out.println("create:" + fileName);
- IndexOutput output = dir.createOutput(fileName, newIOContext(random()));
- output.close();
+
+ AtomicBoolean stop = new AtomicBoolean();
+ Thread writer = new Thread(() -> {
+ try {
+ Random rnd = new Random(RandomizedTest.randomLong() + 1);
+ for (int i = 0, max = RandomizedTest.randomIntBetween(500, 1000); i < max; i++) {
+ String fileName = "file-" + i;
+ try (IndexOutput output = dir.createOutput(fileName, newIOContext(random()))) {
+ // Add some lags so that the other thread can read the content of the directory.
+ Thread.yield();
+ }
assertTrue(slowFileExists(dir, fileName));
- } catch (IOException e) {
- throw new RuntimeException(e);
}
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ } finally {
+ stop.set(true);
}
- }
- };
-
- class TheThread2 extends Thread {
- private String name;
- private volatile boolean stop;
-
- public TheThread2(String name) {
- this.name = name;
- }
-
- @Override
- public void run() {
- while (stop == false) {
- try {
- String[] files = dir.listAll();
- for (String file : files) {
- if (!file.startsWith(name)) {
- continue;
- }
- //System.out.println("file:" + file);
- try {
- IndexInput input = dir.openInput(file, newIOContext(random()));
- input.close();
- } catch (FileNotFoundException | NoSuchFileException e) {
- // ignore
- } catch (IOException e) {
- if (e.getMessage() != null && e.getMessage().contains("still open for writing")) {
- // ignore
- } else {
- throw new RuntimeException(e);
+ });
+
+ Thread reader = new Thread(() -> {
+ try {
+ Random rnd = new Random(RandomizedTest.randomLong());
+ while (!stop.get()) {
+ String [] files = Arrays.stream(dir.listAll())
+ .filter(name -> !ExtrasFS.isExtra(name)) // Ignore anything from ExtraFS.
+ .toArray(String[]::new);
+
+ if (files.length > 0) {
+ do {
+ String file = RandomPicks.randomFrom(rnd, files);
+ try (IndexInput input = dir.openInput(file, newIOContext(random()))) {
+ // Just open, nothing else.
+ } catch (AccessDeniedException e) {
+ // Access denied is allowed for files for which the output is still open.
+ // Since we don't synchronize with the writer thread, just ignore it.
+ } catch (IOException e) {
+ throw new UncheckedIOException("Something went wrong when opening: " + file, e);
}
- }
- if (random().nextBoolean()) {
- break;
- }
+ } while (rnd.nextInt(3) != 0); // Sometimes break and list files again.
}
- } catch (IOException e) {
- throw new RuntimeException(e);
}
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
}
- }
- };
-
- TheThread theThread = new TheThread("t1");
- TheThread2 theThread2 = new TheThread2("t2");
- theThread.start();
- theThread2.start();
-
- theThread.join();
-
- // after first thread is done, no sense in waiting on thread 2
- // to listFiles() and loop over and over
- theThread2.stop = true;
- theThread2.join();
-
- dir.close();
+ });
+
+ reader.start();
+ writer.start();
+
+ writer.join();
+ reader.join();
+ }
}
- /** LUCENE-1468: once we create an output, we should see
- * it in the dir listing and be able to open it with
- * openInput. */
- public void testDirectoryFilter() throws IOException {
- String name = "file";
- Directory dir = getDirectory(createTempDir("testDirectoryFilter"));
- try {
+ /**
+ * LUCENE-1468: once we create an output, we should see
+ * it in the dir listing.
+ */
+ public void testFileExistsInListAfterCreated() throws IOException {
+ try (Directory dir = getDirectory(createTempDir("testFileExistsInListAfterCreated"))) {
+ String name = "file";
dir.createOutput(name, newIOContext(random())).close();
assertTrue(slowFileExists(dir, name));
assertTrue(Arrays.asList(dir.listAll()).contains(name));
- } finally {
- dir.close();
}
}
// LUCENE-2852
public void testSeekToEOFThenBack() throws Exception {
- Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"));
-
- IndexOutput o = dir.createOutput("out", newIOContext(random()));
- byte[] bytes = new byte[3*RAMOutputStream.BUFFER_SIZE];
- o.writeBytes(bytes, 0, bytes.length);
- o.close();
-
- IndexInput i = dir.openInput("out", newIOContext(random()));
- i.seek(2*RAMOutputStream.BUFFER_SIZE-1);
- i.seek(3*RAMOutputStream.BUFFER_SIZE);
- i.seek(RAMOutputStream.BUFFER_SIZE);
- i.readBytes(bytes, 0, 2*RAMOutputStream.BUFFER_SIZE);
- i.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"))) {
+ int bufferLength = 1024;
+ IndexOutput o = dir.createOutput("out", newIOContext(random()));
+ byte[] bytes = new byte[3 * bufferLength];
+ o.writeBytes(bytes, 0, bytes.length);
+ o.close();
+
+ IndexInput i = dir.openInput("out", newIOContext(random()));
+ i.seek(2 * bufferLength - 1);
+ i.seek(3 * bufferLength);
+ i.seek(bufferLength);
+ i.readBytes(bytes, 0, 2 * bufferLength);
+ i.close();
+ }
}
// LUCENE-1196
public void testIllegalEOF() throws Exception {
- Directory dir = getDirectory(createTempDir("testIllegalEOF"));
- IndexOutput o = dir.createOutput("out", newIOContext(random()));
- byte[] b = new byte[1024];
- o.writeBytes(b, 0, 1024);
- o.close();
- IndexInput i = dir.openInput("out", newIOContext(random()));
- i.seek(1024);
- i.close();
- dir.close();
+ try (Directory dir = getDirectory(createTempDir("testIllegalEOF"))) {
+ IndexOutput o = dir.createOutput("out", newIOContext(random()));
+ byte[] b = new byte[1024];
+ o.writeBytes(b, 0, 1024);
+ o.close();
+ IndexInput i = dir.openInput("out", newIOContext(random()));
+ i.seek(1024);
+ i.close();
+ }
}
public void testSeekPastEOF() throws Exception {
- Directory dir = getDirectory(createTempDir("testSeekPastEOF"));
- IndexOutput o = dir.createOutput("out", newIOContext(random()));
- final int len = random().nextInt(2048);
- byte[] b = new byte[len];
- o.writeBytes(b, 0, len);
- o.close();
- IndexInput i = dir.openInput("out", newIOContext(random()));
- expectThrows(EOFException.class, () -> {
- i.seek(len + random().nextInt(2048));
- i.readByte();
- });
+ try (Directory dir = getDirectory(createTempDir("testSeekPastEOF"))) {
+ IndexOutput o = dir.createOutput("out", newIOContext(random()));
+ final int len = random().nextInt(2048);
+ byte[] b = new byte[len];
+ o.writeBytes(b, 0, len);
+ o.close();
+ IndexInput i = dir.openInput("out", newIOContext(random()));
+ expectThrows(EOFException.class, () -> {
+ i.seek(len + random().nextInt(2048));
+ i.readByte();
+ });
- i.close();
- dir.close();
+ i.close();
+ }
}
public void testSliceOutOfBounds() throws Exception {
- Directory dir = getDirectory(createTempDir("testSliceOutOfBounds"));
- IndexOutput o = dir.createOutput("out", newIOContext(random()));
- final int len = random().nextInt(2040) + 8;
- byte[] b = new byte[len];
- o.writeBytes(b, 0, len);
- o.close();
- IndexInput i = dir.openInput("out", newIOContext(random()));
- expectThrows(IllegalArgumentException.class, () -> {
- i.slice("slice1", 0, len + 1);
- });
+ try (Directory dir = getDirectory(createTempDir("testSliceOutOfBounds"))) {
+ IndexOutput o = dir.createOutput("out", newIOContext(random()));
+ final int len = random().nextInt(2040) + 8;
+ byte[] b = new byte[len];
+ o.writeBytes(b, 0, len);
+ o.close();
+ IndexInput i = dir.openInput("out", newIOContext(random()));
+ expectThrows(IllegalArgumentException.class, () -> {
+ i.slice("slice1", 0, len + 1);
+ });
- expectThrows(IllegalArgumentException.class, () -> {
- i.slice("slice2", -1, len);
- });
+ expectThrows(IllegalArgumentException.class, () -> {
+ i.slice("slice2", -1, len);
+ });
- IndexInput slice = i.slice("slice3", 4, len / 2);
- expectThrows(IllegalArgumentException.class, () -> {
- slice.slice("slice3sub", 1, len / 2);
- });
+ IndexInput slice = i.slice("slice3", 4, len / 2);
+ expectThrows(IllegalArgumentException.class, () -> {
+ slice.slice("slice3sub", 1, len / 2);
+ });
- i.close();
- dir.close();
+ i.close();
+ }
}
-
+
// LUCENE-3382 -- make sure we get exception if the directory really does not exist.
public void testNoDir() throws Throwable {
Path tempDir = createTempDir("doesnotexist");
IOUtils.rm(tempDir);
- Directory dir = getDirectory(tempDir);
- try {
- DirectoryReader.open(dir);
- fail("did not hit expected exception");
- } catch (NoSuchFileException | IndexNotFoundException nsde) {
- // expected
+ try (Directory dir = getDirectory(tempDir)) {
+ expectThrowsAnyOf(Arrays.asList(NoSuchFileException.class, IndexNotFoundException.class), () -> {
+ DirectoryReader.open(dir);
+ });
}
- dir.close();
}
public void testCopyBytes() throws Exception {
- testCopyBytes(getDirectory(createTempDir("testCopyBytes")));
- }
-
- private static byte value(int idx) {
- return (byte) ((idx % 256) * (1 + (idx / 256)));
- }
-
- public static void testCopyBytes(Directory dir) throws Exception {
-
- // make random file
- IndexOutput out = dir.createOutput("test", newIOContext(random()));
- byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)];
- final int size = TestUtil.nextInt(random(), 1, 1777777);
- int upto = 0;
- int byteUpto = 0;
- while (upto < size) {
- bytes[byteUpto++] = value(upto);
- upto++;
- if (byteUpto == bytes.length) {
- out.writeBytes(bytes, 0, bytes.length);
- byteUpto = 0;
- }
- }
-
- out.writeBytes(bytes, 0, byteUpto);
- assertEquals(size, out.getFilePointer());
- out.close();
- assertEquals(size, dir.fileLength("test"));
-
- // copy from test -> test2
- final IndexInput in = dir.openInput("test", newIOContext(random()));
-
- out = dir.createOutput("test2", newIOContext(random()));
-
- upto = 0;
- while (upto < size) {
- if (random().nextBoolean()) {
- out.writeByte(in.readByte());
+ try (Directory dir = getDirectory(createTempDir("testCopyBytes"))) {
+ IndexOutput out = dir.createOutput("test", newIOContext(random()));
+ byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)];
+ final int size = TestUtil.nextInt(random(), 1, 1777777);
+ int upto = 0;
+ int byteUpto = 0;
+ while (upto < size) {
+ bytes[byteUpto++] = value(upto);
upto++;
- } else {
- final int chunk = Math.min(
- TestUtil.nextInt(random(), 1, bytes.length), size - upto);
- out.copyBytes(in, chunk);
- upto += chunk;
+ if (byteUpto == bytes.length) {
+ out.writeBytes(bytes, 0, bytes.length);
+ byteUpto = 0;
+ }
}
- }
- assertEquals(size, upto);
- out.close();
- in.close();
-
- // verify
- IndexInput in2 = dir.openInput("test2", newIOContext(random()));
- upto = 0;
- while (upto < size) {
- if (random().nextBoolean()) {
- final byte v = in2.readByte();
- assertEquals(value(upto), v);
- upto++;
- } else {
- final int limit = Math.min(
- TestUtil.nextInt(random(), 1, bytes.length), size - upto);
- in2.readBytes(bytes, 0, limit);
- for (int byteIdx = 0; byteIdx < limit; byteIdx++) {
- assertEquals(value(upto), bytes[byteIdx]);
+
+ out.writeBytes(bytes, 0, byteUpto);
+ assertEquals(size, out.getFilePointer());
+ out.close();
+ assertEquals(size, dir.fileLength("test"));
+
+ // copy from test -> test2
+ final IndexInput in = dir.openInput("test", newIOContext(random()));
+
+ out = dir.createOutput("test2", newIOContext(random()));
+
+ upto = 0;
+ while (upto < size) {
+ if (random().nextBoolean()) {
+ out.writeByte(in.readByte());
+ upto++;
+ } else {
+ final int chunk = Math.min(
+ TestUtil.nextInt(random(), 1, bytes.length), size - upto);
+ out.copyBytes(in, chunk);
+ upto += chunk;
+ }
+ }
+ assertEquals(size, upto);
+ out.close();
+ in.close();
+
+ // verify
+ IndexInput in2 = dir.openInput("test2", newIOContext(random()));
+ upto = 0;
+ while (upto < size) {
+ if (random().nextBoolean()) {
+ final byte v = in2.readByte();
+ assertEquals(value(upto), v);
upto++;
+ } else {
+ final int limit = Math.min(
+ TestUtil.nextInt(random(), 1, bytes.length), size - upto);
+ in2.readBytes(bytes, 0, limit);
+ for (int byteIdx = 0; byteIdx < limit; byteIdx++) {
+ assertEquals(value(upto), bytes[byteIdx]);
+ upto++;
+ }
}
}
+ in2.close();
+
+ dir.deleteFile("test");
+ dir.deleteFile("test2");
}
- in2.close();
-
- dir.deleteFile("test");
- dir.deleteFile("test2");
- dir.close();
}
-
+
+ private static byte value(int idx) {
+ return (byte) ((idx % 256) * (1 + (idx / 256)));
+ }
+
// LUCENE-3541
public void testCopyBytesWithThreads() throws Exception {
- testCopyBytesWithThreads(getDirectory(createTempDir("testCopyBytesWithThreads")));
- }
+ try (Directory d = getDirectory(createTempDir("testCopyBytesWithThreads"))) {
+ byte data[] = RandomBytes.randomBytesOfLengthBetween(random(), 101, 10000);
- public static void testCopyBytesWithThreads(Directory d) throws Exception {
- int datalen = TestUtil.nextInt(random(), 101, 10000);
- byte data[] = new byte[datalen];
- random().nextBytes(data);
-
- IndexOutput output = d.createOutput("data", IOContext.DEFAULT);
- output.writeBytes(data, 0, datalen);
- output.close();
-
- IndexInput input = d.openInput("data", IOContext.DEFAULT);
- IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT);
- // copy our 100-byte header
- outputHeader.copyBytes(input, 100);
- outputHeader.close();
-
- // now make N copies of the remaining bytes
- CopyThread copies[] = new CopyThread[10];
- for (int i = 0; i < copies.length; i++) {
- copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT));
- }
-
- for (int i = 0; i < copies.length; i++) {
- copies[i].start();
- }
-
- for (int i = 0; i < copies.length; i++) {
- copies[i].join();
- }
-
- for (int i = 0; i < copies.length; i++) {
- IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT);
- byte[] dataCopy = new byte[datalen];
- System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing
- copiedData.readBytes(dataCopy, 100, datalen-100);
- assertArrayEquals(data, dataCopy);
- copiedData.close();
- }
- input.close();
- d.close();
-
- }
-
- static class CopyThread extends Thread {
- final IndexInput src;
- final IndexOutput dst;
-
- CopyThread(IndexInput src, IndexOutput dst) {
- this.src = src;
- this.dst = dst;
- }
+ IndexOutput output = d.createOutput("data", IOContext.DEFAULT);
+ output.writeBytes(data, 0, data.length);
+ output.close();
- @Override
- public void run() {
- try {
- dst.copyBytes(src, src.length()-100);
- dst.close();
- } catch (IOException ex) {
- throw new RuntimeException(ex);
+ IndexInput input = d.openInput("data", IOContext.DEFAULT);
+ IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT);
+ // copy our 100-byte header
+ outputHeader.copyBytes(input, 100);
+ outputHeader.close();
+
+ // now make N copies of the remaining bytes
+ int threads = 10;
+ CyclicBarrier start = new CyclicBarrier(threads);
+ Thread copies [] = IntStream.range(0, threads)
+ .mapToObj((i) -> {
+ IndexInput src = input.clone();
+ Thread t = new Thread(() -> {
+ try {
+ start.await();
+ IndexOutput dst = d.createOutput("copy" + i, IOContext.DEFAULT);
+ dst.copyBytes(src, src.length() - 100);
+ dst.close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ t.start();
+ return t;
+ })
+ .toArray(Thread[]::new);
+
+ for (Thread t : copies) {
+ t.join();
}
+
+ for (int i = 0; i < threads; i++) {
+ try (IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT)) {
+ byte[] dataCopy = new byte[data.length];
+ System.arraycopy(data, 0, dataCopy, 0, 100);
+ copiedData.readBytes(dataCopy, 100, data.length - 100);
+ assertArrayEquals(data, dataCopy);
+ }
+ }
+ input.close();
}
}
-
+
// this test backdoors the directory via the filesystem. so it must actually use the filesystem
- // TODO: somehow change this test to
+ // TODO: somehow change this test to
public void testFsyncDoesntCreateNewFiles() throws Exception {
Path path = createTempDir("nocreate");
- Directory fsdir = getDirectory(path);
-
- // this test backdoors the directory via the filesystem. so it must be an FSDir (for now)
- // TODO: figure a way to test this better/clean it up. E.g. we should be testing for FileSwitchDir,
- // if it's using two FSdirs and so on
- if (fsdir instanceof FSDirectory == false) {
- fsdir.close();
- assumeTrue("test only works for FSDirectory subclasses", false);
- }
-
- // write a file
- IndexOutput out = fsdir.createOutput("afile", newIOContext(random()));
- out.writeString("boo");
- out.close();
-
- // delete it
- Files.delete(path.resolve("afile"));
-
- int fileCount = fsdir.listAll().length;
-
- // fsync it
- try {
- fsdir.sync(Collections.singleton("afile"));
- fail("didn't get expected exception, instead fsync created new files: " + Arrays.asList(fsdir.listAll()));
- } catch (FileNotFoundException | NoSuchFileException expected) {
- // ok
+ try (Directory fsdir = getDirectory(path)) {
+ // this test backdoors the directory via the filesystem. so it must be an FSDir (for now)
+ // TODO: figure a way to test this better/clean it up. E.g. we should be testing for FileSwitchDir,
+ // if it's using two FSdirs and so on
+ if (fsdir instanceof FSDirectory == false) {
+ assumeTrue("test only works for FSDirectory subclasses", false);
+ return;
+ }
+
+ // create a file
+ IndexOutput out = fsdir.createOutput("afile", newIOContext(random()));
+ out.writeString("boo");
+ out.close();
+
+ // delete it in the file system.
+ Files.delete(path.resolve("afile"));
+
+ int fileCount = fsdir.listAll().length;
+
+ // fsync it
+ expectThrowsAnyOf(Arrays.asList(FileNotFoundException.class, NoSuchFileException.class), () -> {
+ fsdir.sync(Collections.singleton("afile"));
+ });
+
+ // no new files created
+ assertEquals(fileCount, fsdir.listAll().length);
}
-
- // no new files created
- assertEquals(fileCount, fsdir.listAll().length);
-
- fsdir.close();
}
// random access APIs
-
+
public void testRandomLong() throws Exception {
- Directory dir = getDirectory(createTempDir("testLongs"));
- IndexOutput output = dir.createOutput("longs", newIOContext(random()));
- int num = TestUtil.nextInt(random(), 50, 3000);
- long longs[] = new long[num];
- for (int i = 0; i < longs.length; i++) {
- longs[i] = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE);
- output.writeLong(longs[i]);
- }
- output.close();
-
- // slice
- IndexInput input = dir.openInput("longs", newIOContext(random()));
- RandomAccessInput slice = input.randomAccessSlice(0, input.length());
- for (int i = 0; i < longs.length; i++) {
- assertEquals(longs[i], slice.readLong(i * 8));
- }
-
- // subslices
- for (int i = 1; i < longs.length; i++) {
- long offset = i * 8;
- RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
- for (int j = i; j < longs.length; j++) {
- assertEquals(longs[j], subslice.readLong((j - i) * 8));
+ try (Directory dir = getDirectory(createTempDir("testLongs"))) {
+ IndexOutput output = dir.createOutput("longs", newIOContext(random()));
+ int num = TestUtil.nextInt(random(), 50, 3000);
+ long longs[] = new long[num];
+ for (int i = 0; i < longs.length; i++) {
+ longs[i] = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE);
+ output.writeLong(longs[i]);
}
- }
-
- // with padding
- for (int i = 0; i < 7; i++) {
- String name = "longs-" + i;
- IndexOutput o = dir.createOutput(name, newIOContext(random()));
- byte junk[] = new byte[i];
- random().nextBytes(junk);
- o.writeBytes(junk, junk.length);
- input.seek(0);
- o.copyBytes(input, input.length());
- o.close();
- IndexInput padded = dir.openInput(name, newIOContext(random()));
- RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
- for (int j = 0; j < longs.length; j++) {
- assertEquals(longs[j], whole.readLong(j * 8));
+ output.close();
+
+ // slice
+ IndexInput input = dir.openInput("longs", newIOContext(random()));
+ RandomAccessInput slice = input.randomAccessSlice(0, input.length());
+ for (int i = 0; i < longs.length; i++) {
+ assertEquals(longs[i], slice.readLong(i * 8));
+ }
+
+ // subslices
+ for (int i = 1; i < longs.length; i++) {
+ long offset = i * 8;
+ RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
+ for (int j = i; j < longs.length; j++) {
+ assertEquals(longs[j], subslice.readLong((j - i) * 8));
+ }
+ }
+
+ // with padding
+ for (int i = 0; i < 7; i++) {
+ String name = "longs-" + i;
+ IndexOutput o = dir.createOutput(name, newIOContext(random()));
+ byte junk[] = new byte[i];
+ random().nextBytes(junk);
+ o.writeBytes(junk, junk.length);
+ input.seek(0);
+ o.copyBytes(input, input.length());
+ o.close();
+ IndexInput padded = dir.openInput(name, newIOContext(random()));
+ RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
+ for (int j = 0; j < longs.length; j++) {
+ assertEquals(longs[j], whole.readLong(j * 8));
+ }
+ padded.close();
}
- padded.close();
+
+ input.close();
}
-
- input.close();
- dir.close();
}
public void testRandomInt() throws Exception {
- Directory dir = getDirectory(createTempDir("testInts"));
- IndexOutput output = dir.createOutput("ints", newIOContext(random()));
- int num = TestUtil.nextInt(random(), 50, 3000);
- int ints[] = new int[num];
- for (int i = 0; i < ints.length; i++) {
- ints[i] = random().nextInt();
- output.writeInt(ints[i]);
- }
- output.close();
-
- // slice
- IndexInput input = dir.openInput("ints", newIOContext(random()));
- RandomAccessInput slice = input.randomAccessSlice(0, input.length());
- for (int i = 0; i < ints.length; i++) {
- assertEquals(ints[i], slice.readInt(i * 4));
- }
-
- // subslices
- for (int i = 1; i < ints.length; i++) {
- long offset = i * 4;
- RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
- for (int j = i; j < ints.length; j++) {
- assertEquals(ints[j], subslice.readInt((j - i) * 4));
+ try (Directory dir = getDirectory(createTempDir("testInts"))) {
+ IndexOutput output = dir.createOutput("ints", newIOContext(random()));
+ int num = TestUtil.nextInt(random(), 50, 3000);
+ int ints[] = new int[num];
+ for (int i = 0; i < ints.length; i++) {
+ ints[i] = random().nextInt();
+ output.writeInt(ints[i]);
}
- }
-
- // with padding
- for (int i = 0; i < 7; i++) {
- String name = "ints-" + i;
- IndexOutput o = dir.createOutput(name, newIOContext(random()));
- byte junk[] = new byte[i];
- random().nextBytes(junk);
- o.writeBytes(junk, junk.length);
- input.seek(0);
- o.copyBytes(input, input.length());
- o.close();
- IndexInput padded = dir.openInput(name, newIOContext(random()));
- RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
- for (int j = 0; j < ints.length; j++) {
- assertEquals(ints[j], whole.readInt(j * 4));
+ output.close();
+
+ // slice
+ IndexInput input = dir.openInput("ints", newIOContext(random()));
+ RandomAccessInput slice = input.randomAccessSlice(0, input.length());
+ for (int i = 0; i < ints.length; i++) {
+ assertEquals(ints[i], slice.readInt(i * 4));
+ }
+
+ // subslices
+ for (int i = 1; i < ints.length; i++) {
+ long offset = i * 4;
+ RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
+ for (int j = i; j < ints.length; j++) {
+ assertEquals(ints[j], subslice.readInt((j - i) * 4));
+ }
+ }
+
+ // with padding
+ for (int i = 0; i < 7; i++) {
+ String name = "ints-" + i;
+ IndexOutput o = dir.createOutput(name, newIOContext(random()));
+ byte junk[] = new byte[i];
+ random().nextBytes(junk);
+ o.writeBytes(junk, junk.length);
+ input.seek(0);
+ o.copyBytes(input, input.length());
+ o.close();
+ IndexInput padded = dir.openInput(name, newIOContext(random()));
+ RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
+ for (int j = 0; j < ints.length; j++) {
+ assertEquals(ints[j], whole.readInt(j * 4));
+ }
+ padded.close();
}
- padded.close();
+ input.close();
}
- input.close();
- dir.close();
}
public void testRandomShort() throws Exception {
- Directory dir = getDirectory(createTempDir("testShorts"));
- IndexOutput output = dir.createOutput("shorts", newIOContext(random()));
- int num = TestUtil.nextInt(random(), 50, 3000);
- short shorts[] = new short[num];
- for (int i = 0; i < shorts.length; i++) {
- shorts[i] = (short) random().nextInt();
- output.writeShort(shorts[i]);
- }
- output.close();
-
- // slice
- IndexInput input = dir.openInput("shorts", newIOContext(random()));
- RandomAccessInput slice = input.randomAccessSlice(0, input.length());
- for (int i = 0; i < shorts.length; i++) {
- assertEquals(shorts[i], slice.readShort(i * 2));
- }
-
- // subslices
- for (int i = 1; i < shorts.length; i++) {
- long offset = i * 2;
- RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
- for (int j = i; j < shorts.length; j++) {
- assertEquals(shorts[j], subslice.readShort((j - i) * 2));
+ try (Directory dir = getDirectory(createTempDir("testShorts"))) {
+ IndexOutput output = dir.createOutput("shorts", newIOContext(random()));
+ int num = TestUtil.nextInt(random(), 50, 3000);
+ short shorts[] = new short[num];
+ for (int i = 0; i < shorts.length; i++) {
+ shorts[i] = (short) random().nextInt();
+ output.writeShort(shorts[i]);
}
- }
-
- // with padding
- for (int i = 0; i < 7; i++) {
- String name = "shorts-" + i;
- IndexOutput o = dir.createOutput(name, newIOContext(random()));
- byte junk[] = new byte[i];
- random().nextBytes(junk);
- o.writeBytes(junk, junk.length);
- input.seek(0);
- o.copyBytes(input, input.length());
- o.close();
- IndexInput padded = dir.openInput(name, newIOContext(random()));
- RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
- for (int j = 0; j < shorts.length; j++) {
- assertEquals(shorts[j], whole.readShort(j * 2));
+ output.close();
+
+ // slice
+ IndexInput input = dir.openInput("shorts", newIOContext(random()));
+ RandomAccessInput slice = input.randomAccessSlice(0, input.length());
+ for (int i = 0; i < shorts.length; i++) {
+ assertEquals(shorts[i], slice.readShort(i * 2));
+ }
+
+ // subslices
+ for (int i = 1; i < shorts.length; i++) {
+ long offset = i * 2;
+ RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
+ for (int j = i; j < shorts.length; j++) {
+ assertEquals(shorts[j], subslice.readShort((j - i) * 2));
+ }
}
- padded.close();
+
+ // with padding
+ for (int i = 0; i < 7; i++) {
+ String name = "shorts-" + i;
+ IndexOutput o = dir.createOutput(name, newIOContext(random()));
+ byte junk[] = new byte[i];
+ random().nextBytes(junk);
+ o.writeBytes(junk, junk.length);
+ input.seek(0);
+ o.copyBytes(input, input.length());
+ o.close();
+ IndexInput padded = dir.openInput(name, newIOContext(random()));
+ RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
+ for (int j = 0; j < shorts.length; j++) {
+ assertEquals(shorts[j], whole.readShort(j * 2));
+ }
+ padded.close();
+ }
+ input.close();
}
- input.close();
- dir.close();
}
public void testRandomByte() throws Exception {
- Directory dir = getDirectory(createTempDir("testBytes"));
- IndexOutput output = dir.createOutput("bytes", newIOContext(random()));
- int num = TestUtil.nextInt(random(), 50, 3000);
- byte bytes[] = new byte[num];
- random().nextBytes(bytes);
- for (int i = 0; i < bytes.length; i++) {
- output.writeByte(bytes[i]);
- }
- output.close();
-
- // slice
- IndexInput input = dir.openInput("bytes", newIOContext(random()));
- RandomAccessInput slice = input.randomAccessSlice(0, input.length());
- for (int i = 0; i < bytes.length; i++) {
- assertEquals(bytes[i], slice.readByte(i));
- }
-
- // subslices
- for (int i = 1; i < bytes.length; i++) {
- long offset = i;
- RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
- for (int j = i; j < bytes.length; j++) {
- assertEquals(bytes[j], subslice.readByte(j - i));
+ try (Directory dir = getDirectory(createTempDir("testBytes"))) {
+ IndexOutput output = dir.createOutput("bytes", newIOContext(random()));
+ int num = TestUtil.nextInt(random(), 50, 3000);
+ byte bytes[] = new byte[num];
+ random().nextBytes(bytes);
+ for (int i = 0; i < bytes.length; i++) {
+ output.writeByte(bytes[i]);
}
- }
-
- // with padding
- for (int i = 0; i < 7; i++) {
- String name = "bytes-" + i;
- IndexOutput o = dir.createOutput(name, newIOContext(random()));
- byte junk[] = new byte[i];
- random().nextBytes(junk);
- o.writeBytes(junk, junk.length);
- input.seek(0);
- o.copyBytes(input, input.length());
- o.close();
- IndexInput padded = dir.openInput(name, newIOContext(random()));
- RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
- for (int j = 0; j < bytes.length; j++) {
- assertEquals(bytes[j], whole.readByte(j));
+ output.close();
+
+ // slice
+ IndexInput input = dir.openInput("bytes", newIOContext(random()));
+ RandomAccessInput slice = input.randomAccessSlice(0, input.length());
+ for (int i = 0; i < bytes.length; i++) {
+ assertEquals(bytes[i], slice.readByte(i));
+ }
+
+ // subslices
+ for (int i = 1; i < bytes.length; i++) {
+ long offset = i;
+ RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset);
+ for (int j = i; j < bytes.length; j++) {
+ assertEquals(bytes[j], subslice.readByte(j - i));
+ }
+ }
+
+ // with padding
+ for (int i = 0; i < 7; i++) {
+ String name = "bytes-" + i;
+ IndexOutput o = dir.createOutput(name, newIOContext(random()));
+ byte junk[] = new byte[i];
+ random().nextBytes(junk);
+ o.writeBytes(junk, junk.length);
+ input.seek(0);
+ o.copyBytes(input, input.length());
+ o.close();
+ IndexInput padded = dir.openInput(name, newIOContext(random()));
+ RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i);
+ for (int j = 0; j < bytes.length; j++) {
+ assertEquals(bytes[j], whole.readByte(j));
+ }
+ padded.close();
}
- padded.close();
+ input.close();
}
- input.close();
- dir.close();
}
/** try to stress slices of slices */
public void testSliceOfSlice() throws Exception {
- Directory dir = getDirectory(createTempDir("sliceOfSlice"));
- IndexOutput output = dir.createOutput("bytes", newIOContext(random()));
- final int num;
- if (TEST_NIGHTLY) {
- num = TestUtil.nextInt(random(), 250, 2500);
- } else {
- num = TestUtil.nextInt(random(), 50, 250);
- }
- byte bytes[] = new byte[num];
- random().nextBytes(bytes);
- for (int i = 0; i < bytes.length; i++) {
- output.writeByte(bytes[i]);
- }
- output.close();
-
- IndexInput input = dir.openInput("bytes", newIOContext(random()));
- // seek to a random spot shouldnt impact slicing.
- input.seek(TestUtil.nextLong(random(), 0, input.length()));
- for (int i = 0; i < num; i += 16) {
- IndexInput slice1 = input.slice("slice1", i, num-i);
- assertEquals(0, slice1.getFilePointer());
- assertEquals(num-i, slice1.length());
-
+ try (Directory dir = getDirectory(createTempDir("sliceOfSlice"))) {
+ IndexOutput output = dir.createOutput("bytes", newIOContext(random()));
+ final int num;
+ if (TEST_NIGHTLY) {
+ num = TestUtil.nextInt(random(), 250, 2500);
+ } else {
+ num = TestUtil.nextInt(random(), 50, 250);
+ }
+ byte bytes[] = new byte[num];
+ random().nextBytes(bytes);
+ for (int i = 0; i < bytes.length; i++) {
+ output.writeByte(bytes[i]);
+ }
+ output.close();
+
+ IndexInput input = dir.openInput("bytes", newIOContext(random()));
// seek to a random spot shouldnt impact slicing.
- slice1.seek(TestUtil.nextLong(random(), 0, slice1.length()));
- for (int j = 0; j < slice1.length(); j += 16) {
- IndexInput slice2 = slice1.slice("slice2", j, num-i-j);
- assertEquals(0, slice2.getFilePointer());
- assertEquals(num-i-j, slice2.length());
- byte data[] = new byte[num];
- System.arraycopy(bytes, 0, data, 0, i+j);
- if (random().nextBoolean()) {
- // read the bytes for this slice-of-slice
- slice2.readBytes(data, i+j, num-i-j);
- } else {
- // seek to a random spot in between, read some, seek back and read the rest
- long seek = TestUtil.nextLong(random(), 0, slice2.length());
- slice2.seek(seek);
- slice2.readBytes(data, (int)(i+j+seek), (int)(num-i-j-seek));
- slice2.seek(0);
- slice2.readBytes(data, i+j, (int)seek);
+ input.seek(TestUtil.nextLong(random(), 0, input.length()));
+ for (int i = 0; i < num; i += 16) {
+ IndexInput slice1 = input.slice("slice1", i, num - i);
+ assertEquals(0, slice1.getFilePointer());
+ assertEquals(num - i, slice1.length());
+
+ // seek to a random spot shouldnt impact slicing.
+ slice1.seek(TestUtil.nextLong(random(), 0, slice1.length()));
+ for (int j = 0; j < slice1.length(); j += 16) {
+ IndexInput slice2 = slice1.slice("slice2", j, num - i - j);
+ assertEquals(0, slice2.getFilePointer());
+ assertEquals(num - i - j, slice2.length());
+ byte data[] = new byte[num];
+ System.arraycopy(bytes, 0, data, 0, i + j);
+ if (random().nextBoolean()) {
+ // read the bytes for this slice-of-slice
+ slice2.readBytes(data, i + j, num - i - j);
+ } else {
+ // seek to a random spot in between, read some, seek back and read the rest
+ long seek = TestUtil.nextLong(random(), 0, slice2.length());
+ slice2.seek(seek);
+ slice2.readBytes(data, (int) (i + j + seek), (int) (num - i - j - seek));
+ slice2.seek(0);
+ slice2.readBytes(data, i + j, (int) seek);
+ }
+ assertArrayEquals(bytes, data);
}
- assertArrayEquals(bytes, data);
}
+
+ input.close();
}
-
- input.close();
- dir.close();
}
/**
@@ -1023,82 +966,93 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
* will correctly increment the file pointer.
*/
public void testLargeWrites() throws IOException {
- Directory dir = getDirectory(createTempDir("largeWrites"));
- IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random()));
-
- byte[] largeBuf = new byte[2048];
- random().nextBytes(largeBuf);
-
- long currentPos = os.getFilePointer();
- os.writeBytes(largeBuf, largeBuf.length);
-
- try {
- assertEquals(currentPos + largeBuf.length, os.getFilePointer());
- } finally {
- os.close();
+ try (Directory dir = getDirectory(createTempDir("largeWrites"))) {
+ IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random()));
+
+ byte[] largeBuf = new byte[2048];
+ random().nextBytes(largeBuf);
+
+ long currentPos = os.getFilePointer();
+ os.writeBytes(largeBuf, largeBuf.length);
+
+ try {
+ assertEquals(currentPos + largeBuf.length, os.getFilePointer());
+ } finally {
+ os.close();
+ }
}
- dir.close();
}
// LUCENE-6084
public void testIndexOutputToString() throws Throwable {
- Directory dir = getDirectory(createTempDir());
- IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random()));
- assertTrue(out.toString(), out.toString().contains("camelCase.txt"));
- out.close();
- dir.close();
- }
-
- public void testDoubleCloseDirectory() throws Throwable {
- Directory dir = getDirectory(createTempDir());
- IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
- out.writeString("testing");
- out.close();
- dir.close();
- dir.close(); // close again
+ try (Directory dir = getDirectory(createTempDir())) {
+ IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random()));
+ assertTrue(out.toString(), out.toString().contains("camelCase.txt"));
+ out.close();
+ }
}
public void testDoubleCloseOutput() throws Throwable {
- Directory dir = getDirectory(createTempDir());
- IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
- out.writeString("testing");
- out.close();
- out.close(); // close again
- dir.close();
+ try (Directory dir = getDirectory(createTempDir())) {
+ IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
+ out.writeString("testing");
+ out.close();
+ out.close(); // close again
+ }
}
public void testDoubleCloseInput() throws Throwable {
- Directory dir = getDirectory(createTempDir());
- IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
- out.writeString("testing");
- out.close();
- IndexInput in = dir.openInput("foobar", newIOContext(random()));
- assertEquals("testing", in.readString());
- in.close();
- in.close(); // close again
- dir.close();
+ try (Directory dir = getDirectory(createTempDir())) {
+ IndexOutput out = dir.createOutput("foobar", newIOContext(random()));
+ out.writeString("testing");
+ out.close();
+ IndexInput in = dir.openInput("foobar", newIOContext(random()));
+ assertEquals("testing", in.readString());
+ in.close();
+ in.close(); // close again
+ }
}
public void testCreateTempOutput() throws Throwable {
- Directory dir = getDirectory(createTempDir());
- List<String> names = new ArrayList<>();
- int iters = atLeast(50);
- for(int iter=0;iter<iters;iter++) {
- IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random()));
- names.add(out.getName());
- out.writeVInt(iter);
- out.close();
+ try (Directory dir = getDirectory(createTempDir())) {
+ List<String> names = new ArrayList<>();
+ int iters = atLeast(50);
+ for (int iter = 0; iter < iters; iter++) {
+ IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random()));
+ names.add(out.getName());
+ out.writeVInt(iter);
+ out.close();
+ }
+ for (int iter = 0; iter < iters; iter++) {
+ IndexInput in = dir.openInput(names.get(iter), newIOContext(random()));
+ assertEquals(iter, in.readVInt());
+ in.close();
+ }
+
+ Set<String> files = Arrays.stream(dir.listAll())
+ .filter(file -> !ExtrasFS.isExtra(file)) // remove any ExtrasFS stuff.
+ .collect(Collectors.toSet());
+
+ assertEquals(new HashSet<String>(names), files);
}
- for(int iter=0;iter<iters;iter++) {
- IndexInput in = dir.openInput(names.get(iter), newIOContext(random()));
- assertEquals(iter, in.readVInt());
- in.close();
+ }
+
+ public void testCreateOutputForExistingFile() throws IOException {
+ try (Directory dir = getDirectory(createTempDir())) {
+ String name = "file";
+ try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) {
+ }
+
+ // Try to create an existing file should fail.
+ expectThrows(FileAlreadyExistsException.class, () -> {
+ try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) {
+ }
+ });
+
+ // Delete file and try to recreate it.
+ dir.deleteFile(name);
+ dir.createOutput(name, IOContext.DEFAULT).close();
}
- Set<String> files = new HashSet<String>(Arrays.asList(dir.listAll()));
- // In case ExtrasFS struck:
- files.remove("extra0");
- assertEquals(new HashSet<String>(names), files);
- dir.close();
}
public void testSeekToEndOfFile() throws IOException {
@@ -1185,7 +1139,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
try (Directory dir = getDirectory(createTempDir())) {
int count = atLeast(20);
Set<String> names = new HashSet<>();
- while(names.size() < count) {
+ while (names.size() < count) {
// create a random filename (segment file name style), so it cannot hit windows problem with special filenames ("con", "com1",...):
String name = IndexFileNames.segmentFileName(TestUtil.randomSimpleString(random(), 1, 6), TestUtil.randomSimpleString(random()), "test");
if (random().nextInt(5) == 1) {
@@ -1201,7 +1155,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
String[] actual = dir.listAll();
String[] expected = actual.clone();
Arrays.sort(expected);
- assertEquals(expected, actual);
+ assertArrayEquals(expected, actual);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
index 0194177..ec0d4ff 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
@@ -19,6 +19,7 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -649,11 +650,11 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
throw new IOException("cannot createOutput after crash");
}
init();
- synchronized(this) {
- if (createdFiles.contains(name) && !name.equals("segments.gen")) {
- throw new IOException("file \"" + name + "\" was already written to");
- }
+
+ if (createdFiles.contains(name)) {
+ throw new FileAlreadyExistsException("File \"" + name + "\" was already written to.");
}
+
if (assertNoDeleteOpenFile && openFiles.containsKey(name)) {
throw new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
}
@@ -666,19 +667,24 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
addFileHandle(io, name, Handle.Output);
openFilesForWrite.add(name);
-
+ return maybeThrottle(name, io);
+
+
+ }
+
+ private IndexOutput maybeThrottle(String name, IndexOutput output) {
// throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
- if (throttling == Throttling.ALWAYS ||
+ if (throttling == Throttling.ALWAYS ||
(throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
}
- return throttledOutput.newFromDelegate(io);
+ return throttledOutput.newFromDelegate(output);
} else {
- return io;
+ return output;
}
}
-
+
@Override
public synchronized IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
maybeThrowDeterministicException();
@@ -704,16 +710,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
addFileHandle(io, name, Handle.Output);
openFilesForWrite.add(name);
- // throttling REALLY slows down tests, so don't do it very often for SOMETIMES.
- if (throttling == Throttling.ALWAYS ||
- (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
- }
- return throttledOutput.newFromDelegate(io);
- } else {
- return io;
- }
+ return maybeThrottle(name, io);
}
private static enum Handle {
@@ -750,9 +747,8 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + in) : new NoSuchFileException(name + " in dir=" + in);
}
- // cannot open a file for input if it's still open for
- // output, except for segments.gen and segments_N
- if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name) && !name.startsWith("segments")) {
+ // cannot open a file for input if it's still open for output.
+ if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name)) {
throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index baf72c1..6390cc8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -64,6 +64,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -2683,6 +2684,38 @@ public abstract class LuceneTestCase extends Assert {
throw new AssertionFailedError(noExceptionMessage);
}
+ /** Checks a specific exception class is thrown by the given runnable, and returns it. */
+ public static <T extends Throwable> T expectThrowsAnyOf(List<Class<? extends T>> expectedTypes, ThrowingRunnable runnable) {
+ if (expectedTypes.isEmpty()) {
+ throw new AssertionError("At least one expected exception type is required?");
+ }
+
+ Throwable thrown = null;
+ try {
+ runnable.run();
+ } catch (Throwable e) {
+ for (Class<? extends T> expectedType : expectedTypes) {
+ if (expectedType.isInstance(e)) {
+ return expectedType.cast(e);
+ }
+ }
+ thrown = e;
+ }
+
+ List<String> exceptionTypes = expectedTypes.stream().map(c -> c.getSimpleName()).collect(Collectors.toList());
+
+ if (thrown != null) {
+ AssertionFailedError assertion = new AssertionFailedError("Unexpected exception type, expected any of " +
+ exceptionTypes +
+ " but got: " + thrown);
+ assertion.initCause(thrown);
+ throw assertion;
+ } else {
+ throw new AssertionFailedError("Expected any of the following exception types: " +
+ exceptionTypes+ " but no exception was thrown.");
+ }
+ }
+
/**
* Checks that specific wrapped and outer exception classes are thrown
* by the given runnable, and returns the wrapped exception.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
index 7973256..2ea6ce0 100644
--- a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
+++ b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java
@@ -40,11 +40,12 @@ public class TestMockDirectoryWrapper extends BaseDirectoryTestCase {
}
// we wrap the directory in slow stuff, so only run nightly
- @Override @Nightly
- public void testThreadSafety() throws Exception {
- super.testThreadSafety();
+ @Override
+ @Nightly
+ public void testThreadSafetyInListAll() throws Exception {
+ super.testThreadSafetyInListAll();
}
-
+
public void testDiskFull() throws IOException {
// test writeBytes
MockDirectoryWrapper dir = newMockDirectory();
[2/2] lucene-solr:master: LUCENE-8415: Clean up Directory contracts
(write-once,
no reads-before-write-completed). Minor test improvements and cleanups.
Posted by dw...@apache.org.
LUCENE-8415: Clean up Directory contracts (write-once, no reads-before-write-completed). Minor test improvements and cleanups.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8892c0d9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8892c0d9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8892c0d9
Branch: refs/heads/master
Commit: 8892c0d9aff9e8b7a6722a50a18908ef575276da
Parents: 9d7aef6
Author: Dawid Weiss <dw...@apache.org>
Authored: Tue Jul 24 08:47:50 2018 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Tue Jul 24 08:47:50 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
.../java/org/apache/lucene/store/Directory.java | 194 +-
.../org/apache/lucene/store/IndexOutput.java | 13 +-
.../apache/lucene/index/TestIndexWriter.java | 16 +-
.../org/apache/lucene/util/bkd/TestBKD.java | 7 +-
.../org/apache/lucene/mockfile/ExtrasFS.java | 18 +-
.../lucene/store/BaseDirectoryTestCase.java | 1652 +++++++++---------
.../lucene/store/MockDirectoryWrapper.java | 40 +-
.../org/apache/lucene/util/LuceneTestCase.java | 33 +
.../lucene/store/TestMockDirectoryWrapper.java | 9 +-
10 files changed, 1005 insertions(+), 979 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 730e24c..e324f2c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -202,6 +202,8 @@ Improvements
* LUCENE-8345, GitHub PR #392: Remove instantiation of redundant wrapper classes for primitives;
add wrapper class constructors to forbiddenapis. (Michael Braun via Uwe Schindler)
+* LUCENE-8415: Clean up Directory contracts and JavaDoc comments. (Dawid Weiss)
+
Other:
* LUCENE-8366: Upgrade to ICU 62.1. Emoji handling now uses Unicode 11's
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/core/src/java/org/apache/lucene/store/Directory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index 0c6101d..505be7f 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -27,102 +27,139 @@ import java.util.Set;
import org.apache.lucene.util.IOUtils;
-/** A Directory is a flat list of files. Files may be written once, when they
- * are created. Once a file is created it may only be opened for read, or
- * deleted. Random access is permitted both when reading and writing.
+/**
+ * A {@code Directory} provides an abstraction layer for storing a
+ * list of files. A directory contains only files (no sub-folder hierarchy).
*
- * <p> Java's i/o APIs not used directly, but rather all i/o is
- * through this API. This permits things such as: <ul>
- * <li> implementation of RAM-based indices;
- * <li> implementation indices stored in a database, via JDBC;
- * <li> implementation of an index as a single file;
- * </ul>
+ * Implementing classes must comply with the following:
*
- * Directory locking is implemented by an instance of {@link
- * LockFactory}.
+ * <ul>
+ * <li>A file in a directory can be created ({@link #createOutput}), appended
+ * to, then closed.</li>
+ * <li>A file open for writing may not be available
+ * for read access until the corresponding {@link IndexOutput} is closed.</li>
+ * <li>Once a file is created it must only be opened for input ({@link #openInput}), or
+ * deleted ({@link #deleteFile}). Calling {@link #createOutput} on an existing file
+ * must throw {@link java.nio.file.FileAlreadyExistsException}.</li>
+ * </ul>
*
+ * @see FSDirectory
+ * @see RAMDirectory
+ * @see FilterDirectory
*/
public abstract class Directory implements Closeable {
-
/**
- * Returns an array of strings, one for each entry in the directory, in sorted (UTF16, java's String.compare) order.
+ * Returns names of all files stored in this directory.
+ * The output must be in sorted (UTF-16, java's {@link String#compareTo}) order.
*
- * @throws IOException in case of IO error
+ * @throws IOException in case of I/O error
*/
public abstract String[] listAll() throws IOException;
- /** Removes an existing file in the directory. */
+ /**
+ * Removes an existing file in the directory.
+ *
+ * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException}
+ * if {@code name} points to a non-existing file.
+ *
+ * @param name the name of an existing file.
+ * @throws IOException in case of I/O error
+ */
public abstract void deleteFile(String name) throws IOException;
/**
- * Returns the length of a file in the directory. This method follows the
- * following contract:
- * <ul>
- * <li>Throws {@link FileNotFoundException} or {@link NoSuchFileException}
- * if the file does not exist.
- * <li>Returns a value ≥0 if the file exists, which specifies its length.
- * </ul>
- *
- * @param name the name of the file for which to return the length.
- * @throws IOException if there was an IO error while retrieving the file's
- * length.
+ * Returns the byte length of a file in the directory.
+ *
+ * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException}
+ * if {@code name} points to a non-existing file.
+ *
+ * @param name the name of an existing file.
+ * @throws IOException in case of I/O error
*/
public abstract long fileLength(String name) throws IOException;
- /** Creates a new, empty file in the directory with the given name.
- Returns a stream writing this file. */
+ /**
+ * Creates a new, empty file in the directory and returns an {@link IndexOutput}
+ * instance for appending data to this file.
+ *
+ * This method must throw {@link java.nio.file.FileAlreadyExistsException} if the file
+ * already exists.
+ *
+ * @param name the name of the file to create.
+ * @throws IOException in case of I/O error
+ */
public abstract IndexOutput createOutput(String name, IOContext context) throws IOException;
- /** Creates a new, empty file for writing in the directory, with a
- * temporary file name including prefix and suffix, ending with the
- * reserved extension <code>.tmp</code>. Use
- * {@link IndexOutput#getName} to see what name was used. */
+ /**
+ * Creates a new, empty, temporary file in the directory and returns an {@link IndexOutput}
+ * instance for appending data to this file.
+ *
+ * The temporary file name (accessible via {@link IndexOutput#getName()}) will start with
+ * {@code prefix}, end with {@code suffix} and have a reserved file extension {@code .tmp}.
+ */
public abstract IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException;
/**
- * Ensure that any writes to these files are moved to
- * stable storage. Lucene uses this to properly commit
- * changes to the index, to prevent a machine/OS crash
+ * Ensures that any writes to these files are moved to
+ * stable storage (made durable).
+ *
+ * Lucene uses this to properly commit changes to the index, to prevent a machine/OS crash
* from corrupting the index.
- * <br>
- * NOTE: Clients may call this method for same files over
- * and over again, so some impls might optimize for that.
- * For other impls the operation can be a noop, for various
- * reasons.
+ *
+ * @see #syncMetaData()
*/
public abstract void sync(Collection<String> names) throws IOException;
-
+
/**
- * Renames {@code source} to {@code dest} as an atomic operation,
- * where {@code dest} does not yet exist in the directory.
- * <p>
- * Notes: This method is used by IndexWriter to publish commits.
- * It is ok if this operation is not truly atomic, for example
- * both {@code source} and {@code dest} can be visible temporarily.
- * It is just important that the contents of {@code dest} appear
- * atomically, or an exception is thrown.
+ * Ensures that directory metadata, such as recent file renames, are moved to stable
+ * storage.
+ *
+ * @see #sync(Collection)
*/
- public abstract void rename(String source, String dest) throws IOException;
+ public abstract void syncMetaData() throws IOException;
/**
- * Ensure that directory metadata, such as recent file renames, are made
- * durable.
+ * Renames {@code source} file to {@code dest} file where
+ * {@code dest} must not already exist in the directory.
+ *
+ * It is permitted for this operation to not be truly atomic, for example
+ * both {@code source} and {@code dest} can be visible temporarily in {@link #listAll()}.
+ * However, the implementation of this method must ensure the content of
+ * {@code dest} appears as the entire {@code source} atomically. So once
+ * {@code dest} is visible for readers, the entire content of previous {@code source}
+ * is visible.
+ *
+ * This method is used by IndexWriter to publish commits.
*/
- public abstract void syncMetaData() throws IOException;
-
- /** Returns a stream reading an existing file.
- * <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException}
- * if the file does not exist.
+ public abstract void rename(String source, String dest) throws IOException;
+
+ /**
+ * Opens a stream for reading an existing file.
+ *
+ * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException}
+ * if {@code name} points to a non-existing file.
+ *
+ * @param name the name of an existing file.
+ * @throws IOException in case of I/O error
*/
public abstract IndexInput openInput(String name, IOContext context) throws IOException;
- /** Returns a stream reading an existing file, computing checksum as it reads */
+ /**
+ * Opens a checksum-computing stream for reading an existing file.
+ *
+ * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException}
+ * if {@code name} points to a non-existing file.
+ *
+ * @param name the name of an existing file.
+ * @throws IOException in case of I/O error
+ */
public ChecksumIndexInput openChecksumInput(String name, IOContext context) throws IOException {
return new BufferedChecksumIndexInput(openInput(name, context));
}
/**
- * Returns an obtained {@link Lock}.
+ * Acquires and returns a {@link Lock} for a file with the given name.
+ *
* @param name the name of the lock file
* @throws LockObtainFailedException (optional specific exception) if the lock could
* not be obtained because it is currently held elsewhere.
@@ -130,31 +167,15 @@ public abstract class Directory implements Closeable {
*/
public abstract Lock obtainLock(String name) throws IOException;
- /** Closes the store. */
+ /**
+ * Closes the directory.
+ */
@Override
public abstract void close() throws IOException;
- @Override
- public String toString() {
- return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode());
- }
-
/**
- * Copies the file <i>src</i> in <i>from</i> to this directory under the new
- * file name <i>dest</i>.
- * <p>
- * If you want to copy the entire source directory to the destination one, you
- * can do so like this:
- *
- * <pre class="prettyprint">
- * Directory to; // the directory to copy to
- * for (String file : dir.listAll()) {
- * to.copyFrom(dir, file, newFile, IOContext.DEFAULT); // newFile can be either file, or a new name
- * }
- * </pre>
- * <p>
- * <b>NOTE:</b> this method does not check whether <i>dest</i> exist and will
- * overwrite it if it does.
+ * Copies an existing {@code src} file from directory {@code from}
+ * to a non-existent file {@code dest} in this directory.
*/
public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
boolean success = false;
@@ -169,14 +190,21 @@ public abstract class Directory implements Closeable {
}
}
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode());
+ }
+
/**
- * @throws AlreadyClosedException if this Directory is closed
+ * Ensures this directory is still open.
+ *
+ * @throws AlreadyClosedException if this directory is closed.
*/
protected void ensureOpen() throws AlreadyClosedException {}
/**
- * Returns the current pending deletions in this directory or an emtpy set
- * if there are no known pending deletions.
+ * Returns a set of files currently pending deletion in this directory.
+ *
* @lucene.internal
*/
public Set<String> getPendingDeletions() throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
index e4881e0..14b843f 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
@@ -20,17 +20,16 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.IOException;
-/** Abstract base class for output to a file in a Directory. A random-access
- * output stream. Used for all Lucene index output operations.
-
- * <p>{@code IndexOutput} may only be used from one thread, because it is not
- * thread safe (it keeps internal state like file position).
-
+/**
+ * A {@link DataOutput} for appending data to a file in a {@link Directory}.
+ *
+ * Instances of this class are <b>not</b> thread-safe.
+ *
* @see Directory
* @see IndexInput
*/
public abstract class IndexOutput extends DataOutput implements Closeable {
-
+
/** Full description of this output, e.g. which class such as {@code FSIndexOutput}, and the full path to the file */
private final String resourceDescription;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 8e6de47..31740f9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -68,6 +68,7 @@ import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.mockfile.ExtrasFS;
import org.apache.lucene.mockfile.FilterPath;
import org.apache.lucene.mockfile.WindowsFS;
import org.apache.lucene.search.DocIdSetIterator;
@@ -2730,7 +2731,7 @@ public class TestIndexWriter extends LuceneTestCase {
IndexWriter w = new IndexWriter(dir, iwc);
w.commit();
reader = w.getReader();
- // we pull this commit to open it again later to check that we fail if a futur file delete is pending
+ // we pull this commit to open it again later to check that we fail if a future file delete is pending
indexCommit = reader.getIndexCommit();
w.close();
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
@@ -2740,7 +2741,7 @@ public class TestIndexWriter extends LuceneTestCase {
dir.deleteFile("segments_2");
assertTrue(dir.getPendingDeletions().size() > 0);
- // make sure we get NFSF if we try to delete and already-pending-delete file:
+ // make sure we get NoSuchFileException if we try to delete and already-pending-delete file:
expectThrows(NoSuchFileException.class, () -> {
dir.deleteFile("segments_2");
});
@@ -2777,7 +2778,7 @@ public class TestIndexWriter extends LuceneTestCase {
assertTrue(dir.getPendingDeletions().size() > 0);
- // make sure we get NFSF if we try to delete and already-pending-delete file:
+ // make sure we get NoSuchFileException if we try to delete and already-pending-delete file:
expectThrows(NoSuchFileException.class, () -> {
dir.deleteFile("segments_1");
});
@@ -3348,12 +3349,11 @@ public class TestIndexWriter extends LuceneTestCase {
private static void assertFiles(IndexWriter writer) throws IOException {
Predicate<String> filter = file -> file.startsWith("segments") == false && file.equals("write.lock") == false;
// remove segment files we don't know if we have committed and what is kept around
- Set<String> segFiles = new HashSet<>(writer.segmentInfos.files(true)).stream()
+ Set<String> segFiles = writer.segmentInfos.files(true).stream()
+ .filter(filter).collect(Collectors.toSet());
+ Set<String> dirFiles = Arrays.stream(writer.getDirectory().listAll())
+ .filter(file -> !ExtrasFS.isExtra(file)) // ExtraFS might add an files, ignore them
.filter(filter).collect(Collectors.toSet());
- Set<String> dirFiles = new HashSet<>(Arrays.asList(writer.getDirectory().listAll()))
- .stream().filter(filter).collect(Collectors.toSet());
- // ExtraFS might add an extra0 file, ignore it
- dirFiles.remove("extra0");
assertEquals(segFiles.size(), dirFiles.size());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index b9dad6f..3cf248b 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -29,6 +29,7 @@ import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.index.PointValues;
+import org.apache.lucene.mockfile.ExtrasFS;
import org.apache.lucene.store.CorruptingIndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
@@ -414,8 +415,10 @@ public class TestBKD extends LuceneTestCase {
}
}
- String[] files = dir.listAll();
- assertTrue("files=" + Arrays.toString(files), files.length == 0 || Arrays.equals(files, new String[] {"extra0"}));
+ String[] files = Arrays.stream(dir.listAll())
+ .filter(file -> !ExtrasFS.isExtra(file))
+ .toArray(String[]::new);
+ assertTrue("files=" + Arrays.toString(files), files.length == 0);
dir.close();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8892c0d9/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java
index 28fc44f..f8bba9c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java
@@ -29,11 +29,15 @@ import java.nio.file.attribute.FileAttribute;
* create special files themselves (.DS_Store, thumbs.db, .nfsXXX, ...),
* so we add them and see what breaks.
* <p>
- * When a directory is created, sometimes a file or directory named
- * "extra0" will be included with it.
- * All other filesystem operations are passed thru as normal.
+ * When a directory is created, sometimes an "extra" file or directory
+ * will be included with it (use {@link #isExtra(String)} to check if it's one
+ * of those files).
+ *
+ * All other filesystem operations are delegated as normal.
*/
public class ExtrasFS extends FilterFileSystemProvider {
+ private final static String EXTRA_FILE_NAME = "extra0";
+
final boolean active;
final boolean createDirectory;
@@ -57,7 +61,7 @@ public class ExtrasFS extends FilterFileSystemProvider {
if (active) {
// lets add a bogus file... if this fails, we don't care, its best effort.
try {
- Path target = dir.resolve("extra0");
+ Path target = dir.resolve(EXTRA_FILE_NAME);
if (createDirectory) {
super.createDirectory(target);
} else {
@@ -73,4 +77,10 @@ public class ExtrasFS extends FilterFileSystemProvider {
// our fake files. But this is tricky because its hooked into several places.
// Currently MDW has a hack so we don't break disk full tests.
+ /**
+ * @return Return true if {@code fileName} is one of the extra files added by this class.
+ */
+ public static boolean isExtra(String fileName) {
+ return fileName.equals(EXTRA_FILE_NAME);
+ }
}