You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/02/10 17:27:29 UTC
[1/9] lucene-solr git commit: LUCENE-6932: RAMDirectory's IndexInput
should always throw EOFE if you seek beyond the end of the file and then try
to read
Repository: lucene-solr
Updated Branches:
refs/heads/branch_5_4 b8f57232f -> 7d52c2523
LUCENE-6932: RAMDirectory's IndexInput should always throw EOFE if you seek beyond the end of the file and then try to read
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1725112 13f79535-47bb-0310-9956-ffa450edef68
Conflicts:
lucene/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ad2c18cd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ad2c18cd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ad2c18cd
Branch: refs/heads/branch_5_4
Commit: ad2c18cd72751cd80c11b4916980fb510eaf8f9f
Parents: b8f5723
Author: Michael McCandless <mi...@apache.org>
Authored: Sun Jan 17 19:28:10 2016 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:45:39 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +++
.../org/apache/lucene/store/RAMInputStream.java | 7 +++--
.../apache/lucene/store/TestRAMDirectory.java | 30 +++++++++++++++++++-
3 files changed, 38 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad2c18cd/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d3fa220..fa48106 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -14,6 +14,10 @@ Bug Fixes
* LUCENE-7019: add two-phase iteration to GeoPointTermQueryConstantScoreWrapper.
(Robert Muir via Nick Knize)
+* LUCENE-6932: RAMDirectory's IndexInput should always throw
+ EOFException if you seek past the end of the file and then try to
+ read (Stéphane Campinas via Mike McCandless)
+
======================= Lucene 5.4.1 =======================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad2c18cd/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
index 6cd2e16..46d5dc1 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
@@ -97,7 +97,7 @@ public class RAMInputStream extends IndexInput implements Cloneable {
if (enforceEOF) {
throw new EOFException("read past EOF: " + this);
} else {
- // Force EOF if a read takes place at this position
+ // Force EOF if a read later takes place at this position
currentBufferIndex--;
bufferPosition = BUFFER_SIZE;
}
@@ -120,7 +120,10 @@ public class RAMInputStream extends IndexInput implements Cloneable {
currentBufferIndex = (int) (pos / BUFFER_SIZE);
switchCurrentBuffer(false);
}
- bufferPosition = (int) (pos % BUFFER_SIZE);
+ if (pos < BUFFER_SIZE * (long) file.numBuffers()) {
+ // do not overwrite bufferPosition if EOF should be thrown on the next read
+ bufferPosition = (int) (pos % BUFFER_SIZE);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad2c18cd/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
index fa3f71e..30ceba5 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
@@ -17,12 +17,14 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.EOFException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -160,4 +162,30 @@ public class TestRAMDirectory extends BaseDirectoryTestCase {
writer.close();
}
-}
+
+ public void testShouldThrowEOFException() throws Exception {
+ final Random random = random();
+
+ try (Directory dir = newDirectory()) {
+ final int len = 16 + random().nextInt(2048) / 16 * 16;
+ final byte[] bytes = new byte[len];
+
+ try (IndexOutput os = dir.createOutput("foo", newIOContext(random))) {
+ os.writeBytes(bytes, bytes.length);
+ }
+
+ try (IndexInput is = dir.openInput("foo", newIOContext(random))) {
+ try {
+ is.seek(0);
+ // Here, I go past EOF.
+ is.seek(len + random().nextInt(2048));
+ // since EOF is not enforced by the previous call in RAMInputStream
+ // this call to readBytes should throw the exception.
+ is.readBytes(bytes, 0, 16);
+ fail("Did not get EOFException");
+ } catch (EOFException eof) {
+ // expected!
+ }
+ }
+ }
+ }}
[2/9] lucene-solr git commit: LUCENE-6932: RAMInputStream now throws
EOFException if you seek beyond the end of the file
Posted by mi...@apache.org.
LUCENE-6932: RAMInputStream now throws EOFException if you seek beyond the end of the file
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1726056 13f79535-47bb-0310-9956-ffa450edef68
Conflicts:
lucene/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9041c1cf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9041c1cf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9041c1cf
Branch: refs/heads/branch_5_4
Commit: 9041c1cfe3a7162b77ba2aeb8ba58985ec167528
Parents: ad2c18c
Author: Michael McCandless <mi...@apache.org>
Authored: Thu Jan 21 18:40:02 2016 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:46:24 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 6 +-
.../org/apache/lucene/store/IndexInput.java | 5 +-
.../org/apache/lucene/store/RAMInputStream.java | 84 +++++++++++---------
.../apache/lucene/store/TestRAMDirectory.java | 10 ++-
.../lucene/store/BaseDirectoryTestCase.java | 46 +++++++++--
5 files changed, 99 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9041c1cf/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fa48106..9795dae 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -14,9 +14,9 @@ Bug Fixes
* LUCENE-7019: add two-phase iteration to GeoPointTermQueryConstantScoreWrapper.
(Robert Muir via Nick Knize)
-* LUCENE-6932: RAMDirectory's IndexInput should always throw
- EOFException if you seek past the end of the file and then try to
- read (Stéphane Campinas via Mike McCandless)
+* LUCENE-6932: RAMDirectory's IndexInput was failing to throw
+ EOFException in some cases (Stéphane Campinas, Adrien Grand via Mike
+ McCandless)
======================= Lucene 5.4.1 =======================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9041c1cf/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
index 6815776..02a1734 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
@@ -63,7 +63,10 @@ public abstract class IndexInput extends DataInput implements Cloneable,Closeabl
*/
public abstract long getFilePointer();
- /** Sets current position in this file, where the next read will occur.
+ /** Sets current position in this file, where the next read will occur. If this is
+ * beyond the end of the file then this will throw {@code EOFException} and then the
+ * stream is in an undetermined state.
+ *
* @see #getFilePointer()
*/
public abstract void seek(long pos) throws IOException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9041c1cf/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
index 46d5dc1..3e86a60 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
@@ -17,14 +17,15 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.lucene.store.RAMOutputStream.BUFFER_SIZE;
/** A memory-resident {@link IndexInput} implementation.
*
* @lucene.internal */
public class RAMInputStream extends IndexInput implements Cloneable {
- static final int BUFFER_SIZE = RAMOutputStream.BUFFER_SIZE;
private final RAMFile file;
private final long length;
@@ -33,7 +34,6 @@ public class RAMInputStream extends IndexInput implements Cloneable {
private int currentBufferIndex;
private int bufferPosition;
- private long bufferStart;
private int bufferLength;
public RAMInputStream(String name, RAMFile f) throws IOException {
@@ -48,10 +48,7 @@ public class RAMInputStream extends IndexInput implements Cloneable {
throw new IOException("RAMInputStream too large length=" + length + ": " + name);
}
- // make sure that we switch to the
- // first needed buffer lazily
- currentBufferIndex = -1;
- currentBuffer = null;
+ setCurrentBuffer();
}
@Override
@@ -66,9 +63,8 @@ public class RAMInputStream extends IndexInput implements Cloneable {
@Override
public byte readByte() throws IOException {
- if (bufferPosition >= bufferLength) {
- currentBufferIndex++;
- switchCurrentBuffer(true);
+ if (bufferPosition == bufferLength) {
+ nextBuffer();
}
return currentBuffer[bufferPosition++];
}
@@ -76,9 +72,9 @@ public class RAMInputStream extends IndexInput implements Cloneable {
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
while (len > 0) {
- if (bufferPosition >= bufferLength) {
- currentBufferIndex++;
- switchCurrentBuffer(true);
+
+ if (bufferPosition == bufferLength) {
+ nextBuffer();
}
int remainInBuffer = bufferLength - bufferPosition;
@@ -90,39 +86,49 @@ public class RAMInputStream extends IndexInput implements Cloneable {
}
}
- private final void switchCurrentBuffer(boolean enforceEOF) throws IOException {
- bufferStart = (long) BUFFER_SIZE * (long) currentBufferIndex;
- if (bufferStart > length || currentBufferIndex >= file.numBuffers()) {
- // end of file reached, no more buffers left
- if (enforceEOF) {
- throw new EOFException("read past EOF: " + this);
- } else {
- // Force EOF if a read later takes place at this position
- currentBufferIndex--;
- bufferPosition = BUFFER_SIZE;
- }
- } else {
- currentBuffer = file.getBuffer(currentBufferIndex);
- bufferPosition = 0;
- long buflen = length - bufferStart;
- bufferLength = buflen > BUFFER_SIZE ? BUFFER_SIZE : (int) buflen;
- }
- }
-
@Override
public long getFilePointer() {
- return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
+ return (long) currentBufferIndex * BUFFER_SIZE + bufferPosition;
}
@Override
public void seek(long pos) throws IOException {
- if (currentBuffer == null || pos < bufferStart || pos >= bufferStart + BUFFER_SIZE) {
- currentBufferIndex = (int) (pos / BUFFER_SIZE);
- switchCurrentBuffer(false);
+ int newBufferIndex = (int) (pos / BUFFER_SIZE);
+
+ if (newBufferIndex != currentBufferIndex) {
+ // we seek'd to a different buffer:
+ currentBufferIndex = newBufferIndex;
+ setCurrentBuffer();
}
- if (pos < BUFFER_SIZE * (long) file.numBuffers()) {
- // do not overwrite bufferPosition if EOF should be thrown on the next read
- bufferPosition = (int) (pos % BUFFER_SIZE);
+
+ bufferPosition = (int) (pos % BUFFER_SIZE);
+
+ // This is not >= because seeking to exact end of file is OK: this is where
+ // you'd also be if you did a readBytes of all bytes in the file)
+ if (getFilePointer() > length()) {
+ throw new EOFException("read past EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
+ }
+ }
+
+ private void nextBuffer() throws IOException {
+ // This is >= because we are called when there is at least 1 more byte to read:
+ if (getFilePointer() >= length()) {
+ throw new EOFException("read past EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
+ }
+ currentBufferIndex++;
+ setCurrentBuffer();
+ assert currentBuffer != null;
+ bufferPosition = 0;
+ }
+
+ private final void setCurrentBuffer() throws IOException {
+ if (currentBufferIndex < file.numBuffers()) {
+ currentBuffer = file.getBuffer(currentBufferIndex);
+ assert currentBuffer != null;
+ long bufferStart = (long) BUFFER_SIZE * (long) currentBufferIndex;
+ bufferLength = (int) Math.min(BUFFER_SIZE, length - bufferStart);
+ } else {
+ currentBuffer = null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9041c1cf/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
index 30ceba5..70cd054 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestRAMDirectory.java
@@ -21,7 +21,6 @@ import java.io.EOFException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
@@ -152,10 +151,12 @@ public class TestRAMDirectory extends BaseDirectoryTestCase {
}
};
}
- for (int i=0; i<numThreads; i++)
+ for (int i=0; i<numThreads; i++) {
threads[i].start();
- for (int i=0; i<numThreads; i++)
+ }
+ for (int i=0; i<numThreads; i++) {
threads[i].join();
+ }
writer.forceMerge(1);
assertEquals(ramDir.sizeInBytes(), ramDir.getRecomputedSizeInBytes());
@@ -188,4 +189,5 @@ public class TestRAMDirectory extends BaseDirectoryTestCase {
}
}
}
- }}
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9041c1cf/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
index 4650ac2..af6b7b5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
@@ -586,15 +586,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"));
IndexOutput o = dir.createOutput("out", newIOContext(random()));
- byte[] bytes = new byte[3*RAMInputStream.BUFFER_SIZE];
+ byte[] bytes = new byte[3*RAMOutputStream.BUFFER_SIZE];
o.writeBytes(bytes, 0, bytes.length);
o.close();
IndexInput i = dir.openInput("out", newIOContext(random()));
- i.seek(2*RAMInputStream.BUFFER_SIZE-1);
- i.seek(3*RAMInputStream.BUFFER_SIZE);
- i.seek(RAMInputStream.BUFFER_SIZE);
- i.readBytes(bytes, 0, 2*RAMInputStream.BUFFER_SIZE);
+ i.seek(2*RAMOutputStream.BUFFER_SIZE-1);
+ i.seek(3*RAMOutputStream.BUFFER_SIZE);
+ i.seek(RAMOutputStream.BUFFER_SIZE);
+ i.readBytes(bytes, 0, 2*RAMOutputStream.BUFFER_SIZE);
i.close();
dir.close();
}
@@ -1165,4 +1165,40 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
in.close(); // close again
dir.close();
}
+
+ public void testSeekToEndOfFile() throws IOException {
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
+ for (int i = 0; i < 1024; ++i) {
+ out.writeByte((byte) 0);
+ }
+ }
+ try (IndexInput in = dir.openInput("a", IOContext.DEFAULT)) {
+ in.seek(100);
+ assertEquals(100, in.getFilePointer());
+ in.seek(1024);
+ assertEquals(1024, in.getFilePointer());
+ }
+ }
+ }
+
+ public void testSeekBeyondEndOfFile() throws IOException {
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
+ for (int i = 0; i < 1024; ++i) {
+ out.writeByte((byte) 0);
+ }
+ }
+ try (IndexInput in = dir.openInput("a", IOContext.DEFAULT)) {
+ in.seek(100);
+ assertEquals(100, in.getFilePointer());
+ try {
+ in.seek(1025);
+ fail("didn't hit expected exception");
+ } catch (EOFException eofe) {
+ // expected
+ }
+ }
+ }
+ }
}
[3/9] lucene-solr git commit: LUCENE-6932: also fix NIOFSIndexInput
to throw EOFE if you seek beyond end of file
Posted by mi...@apache.org.
LUCENE-6932: also fix NIOFSIndexInput to throw EOFE if you seek beyond end of file
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1726231 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4fa82b0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4fa82b0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4fa82b0
Branch: refs/heads/branch_5_4
Commit: b4fa82b0772718d84db3d177f8ce7450be3c51ac
Parents: 9041c1c
Author: Michael McCandless <mi...@apache.org>
Authored: Fri Jan 22 15:21:20 2016 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:46:52 2016 -0500
----------------------------------------------------------------------
.../core/src/java/org/apache/lucene/store/NIOFSDirectory.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4fa82b0/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
index 84f1a7f..b739290 100644
--- a/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
@@ -192,6 +192,10 @@ public class NIOFSDirectory extends FSDirectory {
}
@Override
- protected void seekInternal(long pos) throws IOException {}
+ protected void seekInternal(long pos) throws IOException {
+ if (pos > length()) {
+ throw new EOFException("read past EOF: pos=" + pos + " vs length=" + length() + ": " + this);
+ }
+ }
}
}
[9/9] lucene-solr git commit: LUCENE-6976 SOLR-8541:
BytesTermAttributeImpl.copyTo could NPE. Could be triggered by trying to
highlight a spatial RPT field.
Posted by mi...@apache.org.
LUCENE-6976 SOLR-8541: BytesTermAttributeImpl.copyTo could NPE.
Could be triggered by trying to highlight a spatial RPT field.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1724877 13f79535-47bb-0310-9956-ffa450edef68
Conflicts:
lucene/CHANGES.txt
solr/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7d52c252
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7d52c252
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7d52c252
Branch: refs/heads/branch_5_4
Commit: 7d52c2523c7a4ff70612742b76b934a12b493331
Parents: 96624a6
Author: David Wayne Smiley <ds...@apache.org>
Authored: Fri Jan 15 20:59:14 2016 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 11:06:39 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../tokenattributes/BytesTermAttributeImpl.java | 17 ++++++-
.../tokenattributes/TestBytesRefAttImpl.java | 48 ++++++++++++++++++++
solr/CHANGES.txt | 8 ++++
.../apache/solr/search/TestSolr4Spatial2.java | 17 +++++++
5 files changed, 92 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d52c252/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index cb22e23..c3a321b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -24,6 +24,9 @@ Bug Fixes
* LUCENE-7002: Fixed MultiCollector to not throw a NPE if setScorer is called
after one of the sub collectors is done collecting. (John Wang, Adrien Grand)
+* LUCENE-6976: BytesRefTermAttributeImpl.copyTo NPE'ed if BytesRef was null.
+ Added equals & hashCode, and a new test for these things. (David Smiley)
+
======================= Lucene 5.4.1 =======================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d52c252/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java
index d3f8163..81fa816 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/BytesTermAttributeImpl.java
@@ -17,6 +17,8 @@ package org.apache.lucene.analysis.tokenattributes;
* limitations under the License.
*/
+import java.util.Objects;
+
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
import org.apache.lucene.util.BytesRef;
@@ -48,7 +50,7 @@ public class BytesTermAttributeImpl extends AttributeImpl implements BytesTermAt
@Override
public void copyTo(AttributeImpl target) {
BytesTermAttributeImpl other = (BytesTermAttributeImpl) target;
- other.bytes = BytesRef.deepCopyOf(bytes);
+ other.bytes = bytes == null ? null : BytesRef.deepCopyOf(bytes);
}
@Override
@@ -62,4 +64,17 @@ public class BytesTermAttributeImpl extends AttributeImpl implements BytesTermAt
public void reflectWith(AttributeReflector reflector) {
reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof BytesTermAttributeImpl)) return false;
+ BytesTermAttributeImpl that = (BytesTermAttributeImpl) o;
+ return Objects.equals(bytes, that.bytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(bytes);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d52c252/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java
new file mode 100644
index 0000000..9bf21de
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/analysis/tokenattributes/TestBytesRefAttImpl.java
@@ -0,0 +1,48 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestBytesRefAttImpl extends LuceneTestCase {
+
+ public void testCopyTo() throws Exception {
+ BytesTermAttributeImpl t = new BytesTermAttributeImpl();
+ BytesTermAttributeImpl copy = assertCopyIsEqual(t);
+
+ // first do empty
+ assertEquals(t.getBytesRef(), copy.getBytesRef());
+ assertNull(copy.getBytesRef());
+ // now after setting it
+ t.setBytesRef(new BytesRef("hello"));
+ copy = assertCopyIsEqual(t);
+ assertEquals(t.getBytesRef(), copy.getBytesRef());
+ assertNotSame(t.getBytesRef(), copy.getBytesRef());
+ }
+
+ public static <T extends AttributeImpl> T assertCopyIsEqual(T att) throws Exception {
+ @SuppressWarnings("unchecked")
+ T copy = (T) att.getClass().newInstance();
+ att.copyTo(copy);
+ assertEquals("Copied instance must be equal", att, copy);
+ assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode());
+ return copy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d52c252/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 452be4a..cf12274 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -8,6 +8,14 @@ caching, replication, and a web administration interface.
See http://lucene.apache.org/solr for more information.
+================== 5.4.2 ==================
+
+Bug Fixes
+----------------------
+
+* SOLR-8541: Highlighting a geo RPT field would throw an NPE instead of doing nothing.
+ (Pawel Rog via David Smiley)
+
================== 5.4.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d52c252/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java
index c190310..b163db0 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolr4Spatial2.java
@@ -17,9 +17,13 @@ package org.apache.solr.search;
* limitations under the License.
*/
+import org.apache.lucene.analysis.CachingTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.memory.MemoryIndex;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -160,4 +164,17 @@ public class TestSolr4Spatial2 extends SolrTestCaseJ4 {
return getSearcher().getRawReader().leaves().get(0).reader().getCoreCacheKey();
}
+ @Test// SOLR-8541
+ public void testConstantScoreQueryWithFilterPartOnly() {
+ final String[] doc1 = {"id", "1", "srptgeom", "56.9485,24.0980"};
+ assertU(adoc(doc1));
+ assertU(commit());
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("q", "{!geofilt sfield=\"srptgeom\" pt=\"56.9484,24.0981\" d=100}");
+ params.add("hl", "true");
+ params.add("hl.fl", "srptgeom");
+ assertQ(req(params), "*[count(//doc)=1]", "count(//lst[@name='highlighting']/*)=1");
+ }
+
}
[7/9] lucene-solr git commit: LUCENE-6998: fix a couple places to
better detect truncated index files; improve corruption testing
Posted by mi...@apache.org.
LUCENE-6998: fix a couple places to better detect truncated index files; improve corruption testing
Conflicts:
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointFormat.java
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java
lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java
Conflicts:
lucene/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/df30bc6c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/df30bc6c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/df30bc6c
Branch: refs/heads/branch_5_4
Commit: df30bc6c5b4855fcd95c3660fdd2991d0e9c58bf
Parents: 3100f1b
Author: Mike McCandless <mi...@apache.org>
Authored: Thu Jan 28 05:01:15 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:53:08 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/codecs/CodecUtil.java | 4 +-
.../codecs/lucene50/Lucene50CompoundReader.java | 14 ++
.../index/TestAllFilesCheckIndexHeader.java | 132 +++++++++++++++++++
.../index/TestAllFilesDetectTruncation.java | 127 ++++++++++++++++++
.../index/TestAllFilesHaveChecksumFooter.java | 24 ++--
.../index/TestAllFilesHaveCodecHeader.java | 31 +----
.../lucene/index/TestSwappedIndexFiles.java | 127 ++++++++++++++++++
8 files changed, 422 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9795dae..c93d601 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -18,6 +18,9 @@ Bug Fixes
EOFException in some cases (Stéphane Campinas, Adrien Grand via Mike
McCandless)
+* LUCENE-6998: Fix a couple places to better detect truncated index files
+ as corruption. (Robert Muir, Mike McCandless)
+
======================= Lucene 5.4.1 =======================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
index ca6497a..3ca7569 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
@@ -17,7 +17,6 @@ package org.apache.lucene.codecs;
* limitations under the License.
*/
-
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -397,6 +396,9 @@ public final class CodecUtil {
* @throws IOException if the footer is invalid
*/
public static long retrieveChecksum(IndexInput in) throws IOException {
+ if (in.length() < footerLength()) {
+ throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in);
+ }
in.seek(in.length() - footerLength());
validateFooter(in);
return readCRC(in);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
index 5c14b86..99df7f2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
@@ -68,6 +68,13 @@ final class Lucene50CompoundReader extends Directory {
String entriesFileName = IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.ENTRIES_EXTENSION);
this.entries = readEntries(si.getId(), directory, entriesFileName);
boolean success = false;
+
+ long expectedLength = CodecUtil.indexHeaderLength(Lucene50CompoundFormat.DATA_CODEC, "");
+ for(Map.Entry<String,FileEntry> ent : entries.entrySet()) {
+ expectedLength += ent.getValue().length;
+ }
+ expectedLength += CodecUtil.footerLength();
+
handle = directory.openInput(dataFileName, context);
try {
CodecUtil.checkIndexHeader(handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
@@ -77,6 +84,13 @@ final class Lucene50CompoundReader extends Directory {
// for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
// such as file truncation.
CodecUtil.retrieveChecksum(handle);
+
+ // We also validate length, because e.g. if you strip 16 bytes off the .cfs we otherwise
+ // would not detect it:
+ if (handle.length() != expectedLength) {
+ throw new CorruptIndexException("length should be " + expectedLength + " bytes, but is " + handle.length() + " instead", handle);
+ }
+
success = true;
} finally {
if (!success) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
new file mode 100644
index 0000000..9dffaa8
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java
@@ -0,0 +1,132 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Collections;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that a plain default detects broken index headers early (on opening a reader).
+ */
+public class TestAllFilesCheckIndexHeader extends LuceneTestCase {
+ public void test() throws Exception {
+ Directory dir = newDirectory();
+
+ if (dir instanceof MockDirectoryWrapper) {
+ // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ conf.setCodec(TestUtil.getDefaultCodec());
+
+ // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too:
+ if (random().nextInt(5) != 1) {
+ conf.setUseCompoundFile(false);
+ conf.getMergePolicy().setNoCFSRatio(0.0);
+ }
+
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
+ for (int i = 0; i < 100; i++) {
+ riw.addDocument(docs.nextDoc());
+ if (random().nextInt(7) == 0) {
+ riw.commit();
+ }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("docid", Integer.toString(i)));
+ }
+ if (random().nextInt(15) == 0) {
+ riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
+ }
+ }
+
+ if (TEST_NIGHTLY == false) {
+ riw.forceMerge(1);
+ }
+ riw.close();
+ checkIndexHeader(dir);
+ dir.close();
+ }
+
+ private void checkIndexHeader(Directory dir) throws IOException {
+ for(String name : dir.listAll()) {
+ checkOneFile(dir, name);
+ }
+ }
+
+ private void checkOneFile(Directory dir, String victim) throws IOException {
+ try (BaseDirectoryWrapper dirCopy = newDirectory()) {
+ dirCopy.setCheckIndexOnClose(false);
+ long victimLength = dir.fileLength(victim);
+ int wrongBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength));
+ assert victimLength > 0;
+
+ if (VERBOSE) {
+ System.out.println("TEST: now break file " + victim + " by randomizing first " + wrongBytes + " of " + victimLength);
+ }
+
+ for(String name : dir.listAll()) {
+ if (name.equals(victim) == false) {
+ dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
+ } else {
+ try(IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
+ IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
+ // keeps same file length, but replaces the first wrongBytes with random bytes:
+ byte[] bytes = new byte[wrongBytes];
+ random().nextBytes(bytes);
+ out.writeBytes(bytes, 0, bytes.length);
+ in.seek(wrongBytes);
+ out.copyBytes(in, victimLength - wrongBytes);
+ }
+ }
+ dirCopy.sync(Collections.singleton(name));
+ }
+
+ try {
+ // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files:
+ DirectoryReader.open(dirCopy).close();
+ fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
+ } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
+ // expected
+ }
+
+ // CheckIndex should also fail:
+ try {
+ TestUtil.checkIndex(dirCopy, true, true);
+ fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
+ } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
+ // expected
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
new file mode 100644
index 0000000..bda5857
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java
@@ -0,0 +1,127 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that a plain default detects index file truncation early (on opening a reader).
+ */
+public class TestAllFilesDetectTruncation extends LuceneTestCase {
+ public void test() throws Exception {
+ Directory dir = newDirectory();
+
+ if (dir instanceof MockDirectoryWrapper) {
+ // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
+ ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+ }
+
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ conf.setCodec(TestUtil.getDefaultCodec());
+
+ // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too:
+ if (random().nextInt(5) != 1) {
+ conf.setUseCompoundFile(false);
+ conf.getMergePolicy().setNoCFSRatio(0.0);
+ }
+
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
+ for (int i = 0; i < 100; i++) {
+ riw.addDocument(docs.nextDoc());
+ if (random().nextInt(7) == 0) {
+ riw.commit();
+ }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("docid", Integer.toString(i)));
+ }
+ if (random().nextInt(15) == 0) {
+ riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
+ }
+ }
+ if (TEST_NIGHTLY == false) {
+ riw.forceMerge(1);
+ }
+ riw.close();
+ checkTruncation(dir);
+ dir.close();
+ }
+
+ private void checkTruncation(Directory dir) throws IOException {
+ for(String name : dir.listAll()) {
+ truncateOneFile(dir, name);
+ }
+ }
+
+ private void truncateOneFile(Directory dir, String victim) throws IOException {
+ try (BaseDirectoryWrapper dirCopy = newDirectory()) {
+ dirCopy.setCheckIndexOnClose(false);
+ long victimLength = dir.fileLength(victim);
+ int lostBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength));
+ assert victimLength > 0;
+
+ if (VERBOSE) {
+ System.out.println("TEST: now truncate file " + victim + " by removing " + lostBytes + " of " + victimLength + " bytes");
+ }
+
+ for(String name : dir.listAll()) {
+ if (name.equals(victim) == false) {
+ dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
+ } else {
+ try(IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
+ IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
+ out.copyBytes(in, victimLength - lostBytes);
+ }
+ }
+ dirCopy.sync(Collections.singleton(name));
+ }
+
+ try {
+ // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files:
+ DirectoryReader.open(dirCopy).close();
+ fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
+ } catch (CorruptIndexException | EOFException e) {
+ // expected
+ }
+
+ // CheckIndex should also fail:
+ try {
+ TestUtil.checkIndex(dirCopy, true, true);
+ fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
+ } catch (CorruptIndexException | EOFException e) {
+ // expected
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
index 66eb343..710d20f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
@@ -21,11 +21,9 @@ import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -38,23 +36,19 @@ public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(TestUtil.getDefaultCodec());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
- Document doc = new Document();
- // these fields should sometimes get term vectors, etc
- Field idField = newStringField("id", "", Field.Store.NO);
- Field bodyField = newTextField("body", "", Field.Store.NO);
- Field dvField = new NumericDocValuesField("dv", 5);
- doc.add(idField);
- doc.add(bodyField);
- doc.add(dvField);
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
for (int i = 0; i < 100; i++) {
- idField.setStringValue(Integer.toString(i));
- bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
- riw.addDocument(doc);
+ riw.addDocument(docs.nextDoc());
if (random().nextInt(7) == 0) {
riw.commit();
}
if (random().nextInt(20) == 0) {
- riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ riw.deleteDocuments(new Term("docid", Integer.toString(i)));
+ }
+ if (random().nextInt(15) == 0) {
+ riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
}
}
riw.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
index c2b515d..d726019 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
@@ -23,13 +23,9 @@ import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -43,32 +39,19 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setCodec(TestUtil.getDefaultCodec());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
- Document doc = new Document();
- Field idField = newStringField("id", "", Field.Store.YES);
- Field bodyField = newTextField("body", "", Field.Store.YES);
- FieldType vectorsType = new FieldType(TextField.TYPE_STORED);
- vectorsType.setStoreTermVectors(true);
- vectorsType.setStoreTermVectorPositions(true);
- Field vectorsField = new Field("vectors", "", vectorsType);
- Field dvField = new NumericDocValuesField("dv", 5);
- doc.add(idField);
- doc.add(bodyField);
- doc.add(vectorsField);
- doc.add(dvField);
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
for (int i = 0; i < 100; i++) {
- idField.setStringValue(Integer.toString(i));
- bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
- dvField.setLongValue(random().nextInt(5));
- vectorsField.setStringValue(TestUtil.randomUnicodeString(random()));
- riw.addDocument(doc);
+ riw.addDocument(docs.nextDoc());
if (random().nextInt(7) == 0) {
riw.commit();
}
if (random().nextInt(20) == 0) {
- riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ riw.deleteDocuments(new Term("docid", Integer.toString(i)));
}
if (random().nextInt(15) == 0) {
- riw.updateNumericDocValue(new Term("id"), "dv", Long.valueOf(i));
+ riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i));
}
}
riw.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df30bc6c/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
new file mode 100644
index 0000000..008e9a6
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java
@@ -0,0 +1,127 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that the same file name, but from a different index, is detected as foreign.
+ */
+public class TestSwappedIndexFiles extends LuceneTestCase {
+ public void test() throws Exception {
+ Directory dir1 = newDirectory();
+ Directory dir2 = newDirectory();
+
+ if (dir1 instanceof MockDirectoryWrapper) {
+ // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
+ ((MockDirectoryWrapper) dir1).setEnableVirusScanner(false);
+ }
+ if (dir2 instanceof MockDirectoryWrapper) {
+ // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures:
+ ((MockDirectoryWrapper) dir2).setEnableVirusScanner(false);
+ }
+
+ // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too:
+ boolean useCFS = random().nextInt(5) == 1;
+
+ // Use LineFileDocs so we (hopefully) get most Lucene features
+ // tested, e.g. IntPoint was recently added to it:
+ LineFileDocs docs = new LineFileDocs(random());
+ Document doc = docs.nextDoc();
+ long seed = random().nextLong();
+
+ indexOneDoc(seed, dir1, doc, useCFS);
+ indexOneDoc(seed, dir2, doc, useCFS);
+
+ swapFiles(dir1, dir2);
+ dir1.close();
+ dir2.close();
+ }
+
+ private void indexOneDoc(long seed, Directory dir, Document doc, boolean useCFS) throws IOException {
+ Random random = new Random(seed);
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
+ conf.setCodec(TestUtil.getDefaultCodec());
+
+ if (useCFS == false) {
+ conf.setUseCompoundFile(false);
+ conf.getMergePolicy().setNoCFSRatio(0.0);
+ } else {
+ conf.setUseCompoundFile(true);
+ conf.getMergePolicy().setNoCFSRatio(1.0);
+ }
+
+ RandomIndexWriter w = new RandomIndexWriter(random, dir, conf);
+ w.addDocument(doc);
+ w.close();
+ }
+
+ private void swapFiles(Directory dir1, Directory dir2) throws IOException {
+ for(String name : dir1.listAll()) {
+ if (name.equals(IndexWriter.WRITE_LOCK_NAME)) {
+ continue;
+ }
+ swapOneFile(dir1, dir2, name);
+ }
+ }
+
+ private void swapOneFile(Directory dir1, Directory dir2, String victim) throws IOException {
+ try (BaseDirectoryWrapper dirCopy = newDirectory()) {
+ dirCopy.setCheckIndexOnClose(false);
+
+ // Copy all files from dir1 to dirCopy, except victim which we copy from dir2:
+ for(String name : dir1.listAll()) {
+ if (name.equals(victim) == false) {
+ dirCopy.copyFrom(dir1, name, name, IOContext.DEFAULT);
+ } else {
+ dirCopy.copyFrom(dir2, name, name, IOContext.DEFAULT);
+ }
+ dirCopy.sync(Collections.singleton(name));
+ }
+
+ try {
+ // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files:
+ DirectoryReader.open(dirCopy).close();
+ fail("wrong file " + victim + " not detected");
+ } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
+ // expected
+ }
+
+ // CheckIndex should also fail:
+ try {
+ TestUtil.checkIndex(dirCopy, true, true);
+ fail("wrong file " + victim + " not detected");
+ } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
+ // expected
+ }
+ }
+ }
+}
[4/9] lucene-solr git commit: LUCENE-6932: fix test bug that was not
always using the dir impl being tested;
fix SimpleFSIndexInput to throw EOFException if you seek beyond end of file
Posted by mi...@apache.org.
LUCENE-6932: fix test bug that was not always using the dir impl being tested; fix SimpleFSIndexInput to throw EOFException if you seek beyond end of file
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1726277 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/041cd948
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/041cd948
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/041cd948
Branch: refs/heads/branch_5_4
Commit: 041cd9483ec082bc3848cd400c62d50092fc5016
Parents: b4fa82b
Author: Michael McCandless <mi...@apache.org>
Authored: Fri Jan 22 18:39:32 2016 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:47:03 2016 -0500
----------------------------------------------------------------------
.../src/java/org/apache/lucene/store/SimpleFSDirectory.java | 6 +++++-
.../java/org/apache/lucene/store/BaseDirectoryTestCase.java | 4 ++--
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/041cd948/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
index f30b383..2daf98f 100644
--- a/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
@@ -190,6 +190,10 @@ public class SimpleFSDirectory extends FSDirectory {
}
@Override
- protected void seekInternal(long pos) throws IOException {}
+ protected void seekInternal(long pos) throws IOException {
+ if (pos > length()) {
+ throw new EOFException("read past EOF: pos=" + pos + " vs length=" + length() + ": " + this);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/041cd948/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
index af6b7b5..4748f37 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
@@ -1167,7 +1167,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
public void testSeekToEndOfFile() throws IOException {
- try (Directory dir = newDirectory()) {
+ try (Directory dir = getDirectory(createTempDir())) {
try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
for (int i = 0; i < 1024; ++i) {
out.writeByte((byte) 0);
@@ -1183,7 +1183,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
public void testSeekBeyondEndOfFile() throws IOException {
- try (Directory dir = newDirectory()) {
+ try (Directory dir = getDirectory(createTempDir())) {
try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) {
for (int i = 0; i < 1024; ++i) {
out.writeByte((byte) 0);
[5/9] lucene-solr git commit: LUCENE-6932: also fix RAFIndexInput to
throw EOFE if you seek beyond end of file
Posted by mi...@apache.org.
LUCENE-6932: also fix RAFIndexInput to throw EOFE if you seek beyond end of file
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_5x@1726290 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2512ab6c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2512ab6c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2512ab6c
Branch: refs/heads/branch_5_4
Commit: 2512ab6c1f3089cb8fe534532f0676c3358a5cd4
Parents: 041cd94
Author: Michael McCandless <mi...@apache.org>
Authored: Fri Jan 22 19:00:47 2016 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:47:11 2016 -0500
----------------------------------------------------------------------
lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2512ab6c/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java b/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
index 9c3e9b0..94be104 100644
--- a/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
+++ b/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
@@ -165,7 +165,10 @@ public class RAFDirectory extends FSDirectory {
}
@Override
- protected void seekInternal(long position) {
+ protected void seekInternal(long pos) throws IOException {
+ if (pos > length()) {
+ throw new EOFException("read past EOF: pos=" + pos + " vs length=" + length() + ": " + this);
+ }
}
boolean isFDValid() throws IOException {
[6/9] lucene-solr git commit: LUCENE-6932: improve exception messages;
rename length parameter to sliceLength, and return it as the length, for
clarity
Posted by mi...@apache.org.
LUCENE-6932: improve exception messages; rename length parameter to sliceLength, and return it as the length, for clarity
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3100f1b1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3100f1b1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3100f1b1
Branch: refs/heads/branch_5_4
Commit: 3100f1b187ffaeee35dfbad1d26b5c44e5e4c1f7
Parents: 2512ab6
Author: Michael McCandless <ma...@mikemccandless.com>
Authored: Sun Jan 24 06:25:45 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:47:18 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/lucene/store/RAMInputStream.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3100f1b1/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
index 3e86a60..193a088 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RAMInputStream.java
@@ -104,16 +104,16 @@ public class RAMInputStream extends IndexInput implements Cloneable {
bufferPosition = (int) (pos % BUFFER_SIZE);
// This is not >= because seeking to exact end of file is OK: this is where
- // you'd also be if you did a readBytes of all bytes in the file)
+ // you'd also be if you did a readBytes of all bytes in the file
if (getFilePointer() > length()) {
- throw new EOFException("read past EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
+ throw new EOFException("seek beyond EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
}
}
private void nextBuffer() throws IOException {
// This is >= because we are called when there is at least 1 more byte to read:
if (getFilePointer() >= length()) {
- throw new EOFException("read past EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
+ throw new EOFException("cannot read another byte at EOF: pos=" + getFilePointer() + " vs length=" + length() + ": " + this);
}
currentBufferIndex++;
setCurrentBuffer();
@@ -133,11 +133,11 @@ public class RAMInputStream extends IndexInput implements Cloneable {
}
@Override
- public IndexInput slice(String sliceDescription, final long offset, final long length) throws IOException {
- if (offset < 0 || length < 0 || offset + length > this.length) {
+ public IndexInput slice(String sliceDescription, final long offset, final long sliceLength) throws IOException {
+ if (offset < 0 || sliceLength < 0 || offset + sliceLength > this.length) {
throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + this);
}
- return new RAMInputStream(getFullSliceDescription(sliceDescription), file, offset + length) {
+ return new RAMInputStream(getFullSliceDescription(sliceDescription), file, offset + sliceLength) {
{
seek(0L);
}
@@ -157,7 +157,7 @@ public class RAMInputStream extends IndexInput implements Cloneable {
@Override
public long length() {
- return super.length() - offset;
+ return sliceLength;
}
@Override
[8/9] lucene-solr git commit: LUCENE-7002: Fixed MultiCollector to
not throw a NPE if setScorer is called after one of the sub collectors is
done collecting.
Posted by mi...@apache.org.
LUCENE-7002: Fixed MultiCollector to not throw a NPE if setScorer is called after one of the sub collectors is done collecting.
Conflicts:
lucene/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/96624a67
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/96624a67
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/96624a67
Branch: refs/heads/branch_5_4
Commit: 96624a676f5f2bfe3f267e6c1db889e2fe7a1781
Parents: df30bc6
Author: Adrien Grand <jp...@gmail.com>
Authored: Mon Feb 8 16:41:42 2016 +0100
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Feb 10 10:59:24 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../apache/lucene/search/MultiCollector.java | 3 +-
.../lucene/search/TestMultiCollector.java | 71 ++++++++++++++++++++
3 files changed, 76 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96624a67/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c93d601..cb22e23 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -21,6 +21,9 @@ Bug Fixes
* LUCENE-6998: Fix a couple places to better detect truncated index files
as corruption. (Robert Muir, Mike McCandless)
+* LUCENE-7002: Fixed MultiCollector to not throw a NPE if setScorer is called
+ after one of the sub collectors is done collecting. (John Wang, Adrien Grand)
+
======================= Lucene 5.4.1 =======================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96624a67/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java b/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
index 36202e5..a8f872d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiCollector.java
@@ -152,7 +152,8 @@ public class MultiCollector implements Collector {
if (cacheScores) {
scorer = new ScoreCachingWrappingScorer(scorer);
}
- for (LeafCollector c : collectors) {
+ for (int i = 0; i < numCollectors; ++i) {
+ final LeafCollector c = collectors[i];
c.setScorer(scorer);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96624a67/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
index 7ba1e00..261395f 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
@@ -19,9 +19,12 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
@@ -63,6 +66,27 @@ public class TestMultiCollector extends LuceneTestCase {
}
+ private static class SetScorerCollector extends FilterCollector {
+
+ private final AtomicBoolean setScorerCalled;
+
+ public SetScorerCollector(Collector in, AtomicBoolean setScorerCalled) {
+ super(in);
+ this.setScorerCalled = setScorerCalled;
+ }
+
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+ return new FilterLeafCollector(super.getLeafCollector(context)) {
+ @Override
+ public void setScorer(Scorer scorer) throws IOException {
+ super.setScorer(scorer);
+ setScorerCalled.set(true);
+ }
+ };
+ }
+ }
+
public void testCollectionTerminatedExceptionHandling() throws IOException {
final int iters = atLeast(3);
for (int iter = 0; iter < iters; ++iter) {
@@ -95,4 +119,51 @@ public class TestMultiCollector extends LuceneTestCase {
}
}
+ public void testSetScorerAfterCollectionTerminated() throws IOException {
+ Collector collector1 = new TotalHitCountCollector();
+ Collector collector2 = new TotalHitCountCollector();
+
+ AtomicBoolean setScorerCalled1 = new AtomicBoolean();
+ collector1 = new SetScorerCollector(collector1, setScorerCalled1);
+
+ AtomicBoolean setScorerCalled2 = new AtomicBoolean();
+ collector2 = new SetScorerCollector(collector2, setScorerCalled2);
+
+ collector1 = new TerminateAfterCollector(collector1, 1);
+ collector2 = new TerminateAfterCollector(collector2, 2);
+
+ Scorer scorer = new FakeScorer();
+
+ List<Collector> collectors = Arrays.asList(collector1, collector2);
+ Collections.shuffle(collectors, random());
+ Collector collector = MultiCollector.wrap(collectors);
+
+ LeafCollector leafCollector = collector.getLeafCollector(null);
+ leafCollector.setScorer(scorer);
+ assertTrue(setScorerCalled1.get());
+ assertTrue(setScorerCalled2.get());
+
+ leafCollector.collect(0);
+ leafCollector.collect(1);
+
+ setScorerCalled1.set(false);
+ setScorerCalled2.set(false);
+ leafCollector.setScorer(scorer);
+ assertFalse(setScorerCalled1.get());
+ assertTrue(setScorerCalled2.get());
+
+ try {
+ leafCollector.collect(1);
+ fail();
+ } catch (CollectionTerminatedException e) {
+ // expected
+ }
+
+ setScorerCalled1.set(false);
+ setScorerCalled2.set(false);
+ leafCollector.setScorer(scorer);
+ assertFalse(setScorerCalled1.get());
+ assertFalse(setScorerCalled2.get());
+ }
+
}