You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 10:03:48 UTC
[lucene] 11/18: LUCENE-6932: RAMInputStream now throws EOFException
if you seek beyond the end of the file
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch branch_5_4
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 9041c1cfe3a7162b77ba2aeb8ba58985ec167528
Author: Michael McCandless <mi...@apache.org>
AuthorDate: Thu Jan 21 18:40:02 2016 +0000
LUCENE-6932: RAMInputStream now throws EOFException if you seek beyond the end of the file
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1726056 13f79535-47bb-0310-9956-ffa450edef68
Conflicts:
lucene/CHANGES.txt
---
lucene/CHANGES.txt | 6 +-
.../java/org/apache/lucene/store/IndexInput.java | 5 +-
.../org/apache/lucene/store/RAMInputStream.java | 84 ++++++++++++----------
.../org/apache/lucene/store/TestRAMDirectory.java | 10 +--
.../apache/lucene/store/BaseDirectoryTestCase.java | 46 ++++++++++--
5 files changed, 99 insertions(+), 52 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fa48106..9795dae 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -14,9 +14,9 @@ Bug Fixes
* LUCENE-7019: add two-phase iteration to GeoPointTermQueryConstantScoreWrapper.
(Robert Muir via Nick Knize)
-* LUCENE-6932: RAMDirectory's IndexInput should always throw
- EOFException if you seek past the end of the file and then try to
- read (Stéphane Campinas via Mike McCandless)
+* LUCENE-6932: RAMDirectory's IndexInput was failing to throw
+ EOFException in some cases (Stéphane Campinas, Adrien Grand via Mike
+ McCandless)
======================= Lucene 5.4.1 =======================
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
index 6815776..02a1734 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
@@ -63,7 +63,10 @@ public abstract class IndexInput extends DataInput implements Cloneable,Closeabl
*/
public abstract long getFilePointer();
- /** Sets current position in this file, where the next read will occur.
+ /** Sets current position in this file, where the next read will occur. If this is
+ * beyond the end of the file then this will throw {@code EOFException} and then the
+ * stream is in an undetermined state.
+ *
* @see #getFilePointer()
*/
public abstract void seek(long pos) throws IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
index 46d5dc1..3e86a60 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
@@ -17,14 +17,15 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.lucene.store.RAMOutputStream.BUFFER_SIZE;
/** A memory-resident {@link IndexInput} implementation.
*
* @lucene.internal */
public class RAMInputStream extends IndexInput implements Cloneable {
- static final int BUFFER_SIZE = RAMOutputStream.BUFFER_SIZE;
private final RAMFile file;
private final long length;
@@ -33,7 +34,6 @@ public class RAMInputStream extends IndexInput implements Cloneable {
private int currentBufferIndex;
private int bufferPosition;
- private long bufferStart;
private int bufferLength;
public RAMInputStream(String name, RAMFile f) throws IOException {
@@ -48,10 +48,7 @@ public class RAMInputStream extends IndexInput implements Cloneable {
throw new IOException("RAMInputStream too large length=" + length + ": " + name);
}
- // make sure that we switch to the
- // first needed buffer lazily
- currentBufferIndex = -1;
- currentBuffer = null;
+ setCurrentBuffer();
}
@Override
@@ -66,9 +63,8 @@ public class RAMInputStream extends IndexInput implements Cloneable {
@Override
public byte readByte() throws IOException {
- if (bufferPosition >= bufferLength) {
- currentBufferIndex++;
- switchCurrentBuffer(true);
+ if (bufferPosition == bufferLength) {
+ nextBuffer();
}
return currentBuffer[bufferPosition++];
}
@@ -76,9 +72,9 @@ public class RAMInputStream extends IndexInput implements Cloneable {
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
while (len > 0) {
- if (bufferPosition >= bufferLength) {
- currentBufferIndex++;
- switchCurrentBuffer(true);
+
+ if (bufferPosition == bufferLength) {
+ nextBuffer();
}
int remainInBuffer = bufferLength - bufferPosition;
@@ -90,39 +86,49 @@ public class RAMInputStream extends IndexInput implements Cloneable {
}
}
- private final void switchCurrentBuffer(boolean enforceEOF) throws IOException {
- bufferStart = (long) BUFFER_SIZE * (long) currentBufferIndex;
- if (bufferStart > length || currentBufferIndex >= file.numBuffers()) {
- // end of file reached, no more buffers left
- if (enforceEOF) {
- throw new EOFException("read past EOF: " + this);
- } else {
- // Force EOF if a read later takes place at this position
- currentBufferIndex--;
- bufferPosition = BUFFER_SIZE;
- }
- } else {
- currentBuffer = file.getBuffer(currentBufferIndex);
- bufferPosition = 0;
- long buflen = length - bufferStart;
- bufferLength = buflen > BUFFER_SIZE ? BUFFER_SIZE : (int) buflen;
- }
- }
-
@Override
public long getFilePointer() {
- return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
+ return (long) currentBufferIndex * BUFFER_SIZE + bufferPosition;
}
@Override
public void seek(long pos) throws IOException {
- if (currentBuffer == null || pos < bufferStart || pos >= bufferStart + BUFFER_SIZE) {
- currentBufferIndex = (int) (pos / BUFFER_SIZE);
- switchCurrentBuffer(false);
+ int newBufferIndex = (int) (pos / BUFFER_SIZE);
+
+ if (newBufferIndex != currentBufferIndex) {
+ // we seek'd to a different buffer:
+ currentBufferIndex = newBufferIndex;
+ setCurrentBuffer();
}
- if (pos < BUFFER_SIZE * (long) file.numBuffers()) {
- // do not overwrite bufferPosition if EOF should be thrown on the next read
- bufferPosition = (int) (pos % BUFFER_SIZE);
+
+ bufferPosition = (int) (pos % BUFFER_SIZE);
+
+ // This is not >= because seeking to exact end of file is OK: this is where
+ // you'd also be if you did a readBytes of all bytes in the file)
+ if (getFilePointer() > length()) {
+ throw new EOFException("read past EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
+ }
+ }
+
+ private void nextBuffer() throws IOException {
+ // This is >= because we are called when there is at least 1 more byte to read:
+ if (getFilePointer() >= length()) {
+ throw new EOFException("read past EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
+ }
+ currentBufferIndex++;
+ setCurrentBuffer();
+ assert currentBuffer != null;
+ bufferPosition = 0;
+ }
+
+ private final void setCurrentBuffer() throws IOException {
+ if (currentBufferIndex < file.numBuffers()) {
+ currentBuffer = file.getBuffer(currentBufferIndex);
+ assert currentBuffer != null;
+ long bufferStart = (long) BUFFER_SIZE * (long) currentBufferIndex;
+ bufferLength = (int) Math.min(BUFFER_SIZE, length - bufferStart);
+ } else {
+ currentBuffer = null;
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
index 30ceba5..70cd054 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
@@ -21,7 +21,6 @@ import java.io.EOFException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
@@ -152,10 +151,12 @@ public class TestRAMDirectory extends BaseDirectoryTestCase {
}
};
}
- for (int i=0; i<numThreads; i++)
+ for (int i=0; i<numThreads; i++) {
threads[i].start();
- for (int i=0; i<numThreads; i++)
+ }
+ for (int i=0; i<numThreads; i++) {
threads[i].join();
+ }
writer.forceMerge(1);
assertEquals(ramDir.sizeInBytes(), ramDir.getRecomputedSizeInBytes());
@@ -188,4 +189,5 @@ public class TestRAMDirectory extends BaseDirectoryTestCase {
}
}
}
- }}
+ }
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
index 4650ac2..af6b7b5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
@@ -586,15 +586,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"));
IndexOutput o = dir.createOutput("out", newIOContext(random()));
- byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE];
+ byte[] bytes = new byte[3*RAMOutputStream.BUFFER_SIZE];
o.writeBytes(bytes, 0, bytes.length);
o.close();
IndexInput i = dir.openInput("out", newIOContext(random()));
- i.seek(2*RAMInputStream.BUFFER_SIZE-1);
- i.seek(3*RAMInputStream.BUFFER_SIZE);
- i.seek(RAMInputStream.BUFFER_SIZE);
- i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE);
+ i.seek(2*RAMOutputStream.BUFFER_SIZE-1);
+ i.seek(3*RAMOutputStream.BUFFER_SIZE);
+ i.seek(RAMOutputStream.BUFFER_SIZE);
+ i.readBytes(bytes, 0, 2*RAMOutputStream.BUFFER_SIZE);
i.close();
dir.close();
}
@@ -1165,4 +1165,40 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
in.close(); // close again
dir.close();
}
+
+ public void testSeekToEndOfFile() throws IOException {
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
+ for (int i = 0; i < 1024; ++i) {
+ out.writeByte((byte) 0);
+ }
+ }
+ try (IndexInput in = dir.openInput("a", IOContext.DEFAULT)) {
+ in.seek(100);
+ assertEquals(100, in.getFilePointer());
+ in.seek(1024);
+ assertEquals(1024, in.getFilePointer());
+ }
+ }
+ }
+
+ public void testSeekBeyondEndOfFile() throws IOException {
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
+ for (int i = 0; i < 1024; ++i) {
+ out.writeByte((byte) 0);
+ }
+ }
+ try (IndexInput in = dir.openInput("a", IOContext.DEFAULT)) {
+ in.seek(100);
+ assertEquals(100, in.getFilePointer());
+ try {
+ in.seek(1025);
+ fail("didn't hit expected exception");
+ } catch (EOFException eofe) {
+ // expected
+ }
+ }
+ }
+ }
}