You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/04/14 20:54:11 UTC
svn commit: r1673508 - in /lucene/dev/trunk/lucene: ./
backward-codecs/src/test/org/apache/lucene/index/
core/src/java/org/apache/lucene/codecs/lucene50/
core/src/java/org/apache/lucene/codecs/perfield/
core/src/java/org/apache/lucene/index/ core/src/j...
Author: mikemccand
Date: Tue Apr 14 18:54:10 2015
New Revision: 1673508
URL: http://svn.apache.org/r1673508
Log:
LUCENE-6382: enforce max allowed indexed position
Added:
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java (with props)
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/maxposindex.zip (with props)
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMaxPosition.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappingMultiPostingsEnum.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Version.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Apr 14 18:54:10 2015
@@ -113,6 +113,9 @@ Other
* LUCENE-6399: Benchmark module's QueryMaker.resetInputs should call setConfig
so queries can react to property changes in new rounds. (David Smiley)
+* LUCENE-6382: Lucene now enforces that positions never exceed the
+ maximum value IndexWriter.MAX_POSITION. (Robert Muir, Mike McCandless)
+
Build
* LUCENE-6420: Update forbiddenapis to v1.8 (Uwe Schindler)
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java?rev=1673508&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java (added)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java Tue Apr 14 18:54:10 2015
@@ -0,0 +1,106 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.InputStream;
+import java.nio.file.Path;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+// LUCENE-6382
+public class TestMaxPositionInOldIndex extends LuceneTestCase {
+
+
+ // Save this to BuildMaxPositionIndex.java and follow the compile/run instructions to regenerate the .zip:
+ /*
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+
+// Compile:
+// javac -cp lucene/build/core/lucene-core-5.1.0-SNAPSHOT.jar:lucene/build/test-framework/lucene-test-framework-5.1.0-SNAPSHOT.jar:lucene/build/analysis/common/lucene-analyzers-common-5.1.0-SNAPSHOT.jar BuildMaxPositionIndex.java
+
+// Run:
+// java -cp .:lucene/build/core/lucene-core-5.1.0-SNAPSHOT.jar:lucene/build/test-framework/lucene-test-framework-5.1.0-SNAPSHOT.jar:lucene/build/analysis/common/lucene-analyzers-common-5.1.0-SNAPSHOT.jar:lucene/build/codecs/lucene-codecs-5.1.0-SNAPSHOT.jar BuildMaxPositionIndex
+
+// cd maxposindex
+// zip maxposindex.zip *
+
+public class BuildMaxPositionIndex {
+ public static void main(String[] args) throws IOException {
+ Directory dir = FSDirectory.open(Paths.get("maxposindex"));
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(new WhitespaceAnalyzer()));
+ Document doc = new Document();
+ // This is at position 1:
+ Token t1 = new Token("foo", 0, 3);
+ t1.setPositionIncrement(2);
+ Token t2 = new Token("foo", 4, 7);
+ // This overflows max position:
+ t2.setPositionIncrement(Integer.MAX_VALUE-1);
+ t2.setPayload(new BytesRef(new byte[] { 0x1 } ));
+ doc.add(new TextField("foo", new CannedTokenStream(new Token[] {t1, t2})));
+ iw.addDocument(doc);
+ iw.close();
+ dir.close();
+ }
+}
+ */
+
+ public void testCorruptIndex() throws Exception {
+ Path path = createTempDir("maxposindex");
+ InputStream resource = getClass().getResourceAsStream("maxposindex.zip");
+ assertNotNull("maxposindex not found", resource);
+ TestUtil.unzip(resource, path);
+ BaseDirectoryWrapper dir = newFSDirectory(path);
+ dir.setCheckIndexOnClose(false);
+ try {
+ TestUtil.checkIndex(dir, false, true);
+ fail("corruption was not detected");
+ } catch (RuntimeException re) {
+ // expected
+ assertTrue(re.getMessage().contains("pos 2147483647 > IndexWriter.MAX_POSITION=2147483519"));
+ }
+
+ // Also confirm merging detects this:
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler()));
+ w.addDocument(new Document());
+ try {
+ w.forceMerge(1);
+ } catch (CorruptIndexException cie) {
+ // SerialMergeScheduler
+ assertTrue(cie.getMessage().contains("position=2147483647 is too large (> IndexWriter.MAX_POSITION=2147483519), field=\"foo\" doc=0 (resource=PerFieldPostings(segment=_0 formats=1)"));
+ }
+
+ w.close();
+ dir.close();
+ }
+}
+
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/maxposindex.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/maxposindex.zip?rev=1673508&view=auto
==============================================================================
Binary file - no diff available.
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java Tue Apr 14 18:54:10 2015
@@ -36,6 +36,7 @@ import org.apache.lucene.codecs.lucene50
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
@@ -250,6 +251,12 @@ public final class Lucene50PostingsWrite
@Override
public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException {
+ if (position > IndexWriter.MAX_POSITION) {
+ throw new CorruptIndexException("position=" + position + " is too large (> IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION + ")", docOut);
+ }
+ if (position < 0) {
+ throw new CorruptIndexException("position=" + position + " is < 0", docOut);
+ }
posDeltaBuffer[posBufferUpto] = position - lastPosition;
if (writePayloads) {
if (payload == null || payload.length == 0) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Tue Apr 14 18:54:10 2015
@@ -217,6 +217,7 @@ public abstract class PerFieldPostingsFo
private final Map<String,FieldsProducer> fields = new TreeMap<>();
private final Map<String,FieldsProducer> formats = new HashMap<>();
+ private final String segment;
// clone for merge
FieldsReader(FieldsReader other) throws IOException {
@@ -234,6 +235,8 @@ public abstract class PerFieldPostingsFo
assert producer != null;
fields.put(ent.getKey(), producer);
}
+
+ segment = other.segment;
}
public FieldsReader(final SegmentReadState readState) throws IOException {
@@ -267,6 +270,8 @@ public abstract class PerFieldPostingsFo
IOUtils.closeWhileHandlingException(formats.values());
}
}
+
+ this.segment = readState.segmentInfo.name;
}
@Override
@@ -320,7 +325,7 @@ public abstract class PerFieldPostingsFo
@Override
public String toString() {
- return "PerFieldPostings(formats=" + formats.size() + ")";
+ return "PerFieldPostings(segment=" + segment + " formats=" + formats.size() + ")";
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Apr 14 18:54:10 2015
@@ -1322,6 +1322,9 @@ public class CheckIndex implements Close
if (pos < 0) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
}
+ if (pos > IndexWriter.MAX_POSITION) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " > IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION);
+ }
if (pos < lastPos) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Tue Apr 14 18:54:10 2015
@@ -612,8 +612,11 @@ final class DefaultIndexingChain extends
if (invertState.position < invertState.lastPosition) {
if (posIncr == 0) {
throw new IllegalArgumentException("first position increment must be > 0 (got 0) for field '" + field.name() + "'");
+ } else {
+ throw new IllegalArgumentException("position increments (and gaps) must be >= 0 (got " + posIncr + ") for field '" + field.name() + "'");
}
- throw new IllegalArgumentException("position increments (and gaps) must be >= 0 (got " + posIncr + ") for field '" + field.name() + "'");
+ } else if (invertState.position > IndexWriter.MAX_POSITION) {
+ throw new IllegalArgumentException("position " + invertState.position + " is too large for field '" + field.name() + "': max allowed position is " + IndexWriter.MAX_POSITION);
}
invertState.lastPosition = invertState.position;
if (posIncr == 0) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Apr 14 18:54:10 2015
@@ -203,6 +203,9 @@ public class IndexWriter implements Clos
// ArrayUtil.MAX_ARRAY_LENGTH here because this can vary across JVMs:
public static final int MAX_DOCS = Integer.MAX_VALUE - 128;
+ /** Maximum value of the token position in an indexed field. */
+ public static final int MAX_POSITION = Integer.MAX_VALUE - 128;
+
// Use package-private instance var to enforce the limit so testing
// can use less electricity:
private static int actualMaxDocs = MAX_DOCS;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java Tue Apr 14 18:54:10 2015
@@ -46,21 +46,23 @@ public class MappedMultiFields extends F
if (terms == null) {
return null;
} else {
- return new MappedMultiTerms(mergeState, terms);
+ return new MappedMultiTerms(field, mergeState, terms);
}
}
private static class MappedMultiTerms extends FilterTerms {
final MergeState mergeState;
+ final String field;
- public MappedMultiTerms(MergeState mergeState, MultiTerms multiTerms) {
+ public MappedMultiTerms(String field, MergeState mergeState, MultiTerms multiTerms) {
super(multiTerms);
+ this.field = field;
this.mergeState = mergeState;
}
@Override
public TermsEnum iterator() throws IOException {
- return new MappedMultiTermsEnum(mergeState, (MultiTermsEnum) in.iterator());
+ return new MappedMultiTermsEnum(field, mergeState, (MultiTermsEnum) in.iterator());
}
@Override
@@ -86,9 +88,11 @@ public class MappedMultiFields extends F
private static class MappedMultiTermsEnum extends FilterTermsEnum {
final MergeState mergeState;
+ final String field;
- public MappedMultiTermsEnum(MergeState mergeState, MultiTermsEnum multiTermsEnum) {
+ public MappedMultiTermsEnum(String field, MergeState mergeState, MultiTermsEnum multiTermsEnum) {
super(multiTermsEnum);
+ this.field = field;
this.mergeState = mergeState;
}
@@ -110,26 +114,19 @@ public class MappedMultiFields extends F
MappingMultiPostingsEnum mappingDocsAndPositionsEnum;
if (reuse instanceof MappingMultiPostingsEnum) {
- mappingDocsAndPositionsEnum = (MappingMultiPostingsEnum) reuse;
+ MappingMultiPostingsEnum postings = (MappingMultiPostingsEnum) reuse;
+ if (postings.field.equals(this.field)) {
+ mappingDocsAndPositionsEnum = postings;
+ } else {
+ mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(field, mergeState);
+ }
} else {
- mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(mergeState);
+ mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(field, mergeState);
}
MultiPostingsEnum docsAndPositionsEnum = (MultiPostingsEnum) in.postings(liveDocs, mappingDocsAndPositionsEnum.multiDocsAndPositionsEnum, flags);
mappingDocsAndPositionsEnum.reset(docsAndPositionsEnum);
return mappingDocsAndPositionsEnum;
-
-/*
- MappingMultiDocsEnum mappingDocsEnum;
- if (reuse instanceof MappingMultiDocsEnum) {
- mappingDocsEnum = (MappingMultiDocsEnum) reuse;
- } else {
- mappingDocsEnum = new MappingMultiDocsEnum(mergeState);
- }
-
- MultiDocsEnum docsEnum = (MultiDocsEnum) in.docs(liveDocs, mappingDocsEnum.multiDocsEnum, flags);
- mappingDocsEnum.reset(docsEnum);
- return mappingDocsEnum;*/
}
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappingMultiPostingsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappingMultiPostingsEnum.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappingMultiPostingsEnum.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/MappingMultiPostingsEnum.java Tue Apr 14 18:54:10 2015
@@ -39,9 +39,11 @@ final class MappingMultiPostingsEnum ext
int doc = -1;
private MergeState mergeState;
MultiPostingsEnum multiDocsAndPositionsEnum;
+ final String field;
/** Sole constructor. */
- public MappingMultiPostingsEnum(MergeState mergeState) {
+ public MappingMultiPostingsEnum(String field, MergeState mergeState) {
+ this.field = field;
this.mergeState = mergeState;
}
@@ -112,9 +114,17 @@ final class MappingMultiPostingsEnum ext
@Override
public int nextPosition() throws IOException {
- return current.nextPosition();
+ int pos = current.nextPosition();
+ if (pos < 0) {
+ throw new CorruptIndexException("position=" + pos + " is negative, field=\"" + field + " doc=" + doc,
+ mergeState.fieldsProducers[upto].toString());
+ } else if (pos > IndexWriter.MAX_POSITION) {
+ throw new CorruptIndexException("position=" + pos + " is too large (> IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION + "), field=\"" + field + "\" doc=" + doc,
+ mergeState.fieldsProducers[upto].toString());
+ }
+ return pos;
}
-
+
@Override
public int startOffset() throws IOException {
return current.startOffset();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Version.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/Version.java Tue Apr 14 18:54:10 2015
@@ -202,7 +202,7 @@ public final class Version {
/** Returns a new version based on raw numbers
*
* @lucene.internal */
- public static final Version fromBits(int major, int minor, int bugfix) {
+ public static Version fromBits(int major, int minor, int bugfix) {
return new Version(major, minor, bugfix);
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java Tue Apr 14 18:54:10 2015
@@ -29,12 +29,9 @@ import org.apache.lucene.store.BaseDirec
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util.LuceneTestCase.Monster;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
/**
* Test indexes ~82M docs with 52 positions each, so you get > Integer.MAX_VALUE positions
* @lucene.experimental
@@ -60,8 +57,8 @@ public class Test2BPositions extends Luc
MergePolicy mp = w.getConfig().getMergePolicy();
if (mp instanceof LogByteSizeMergePolicy) {
- // 1 petabyte:
- ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
+ // 1 petabyte:
+ ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
}
Document doc = new Document();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Apr 14 18:54:10 2015
@@ -30,10 +30,8 @@ import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -1727,49 +1725,7 @@ public class TestIndexWriterExceptions e
uoe.doFail = false;
d.close();
}
-
- public void testIllegalPositions() throws Exception {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
- Document doc = new Document();
- Token t1 = new Token("foo", 0, 3);
- t1.setPositionIncrement(Integer.MAX_VALUE);
- Token t2 = new Token("bar", 4, 7);
- t2.setPositionIncrement(200);
- TokenStream overflowingTokenStream = new CannedTokenStream(
- new Token[] { t1, t2 }
- );
- Field field = new TextField("foo", overflowingTokenStream);
- doc.add(field);
- try {
- iw.addDocument(doc);
- fail();
- } catch (IllegalArgumentException expected) {
- // expected exception
- }
- iw.close();
- dir.close();
- }
-
- public void testLegalbutVeryLargePositions() throws Exception {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
- Document doc = new Document();
- Token t1 = new Token("foo", 0, 3);
- t1.setPositionIncrement(Integer.MAX_VALUE-500);
- if (random().nextBoolean()) {
- t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
- }
- TokenStream overflowingTokenStream = new CannedTokenStream(
- new Token[] { t1 }
- );
- Field field = new TextField("foo", overflowingTokenStream);
- doc.add(field);
- iw.addDocument(doc);
- iw.close();
- dir.close();
- }
-
+
public void testBoostOmitNorms() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMaxPosition.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMaxPosition.java?rev=1673508&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMaxPosition.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestMaxPosition.java Tue Apr 14 18:54:10 2015
@@ -0,0 +1,103 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+// LUCENE-6382
+public class TestMaxPosition extends LuceneTestCase {
+
+ public void testTooBigPosition() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+ Document doc = new Document();
+ // This is at position 1:
+ Token t1 = new Token("foo", 0, 3);
+ t1.setPositionIncrement(2);
+ if (random().nextBoolean()) {
+ t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
+ }
+ Token t2 = new Token("foo", 4, 7);
+ // This should overflow max:
+ t2.setPositionIncrement(IndexWriter.MAX_POSITION);
+ if (random().nextBoolean()) {
+ t2.setPayload(new BytesRef(new byte[] { 0x1 } ));
+ }
+ doc.add(new TextField("foo", new CannedTokenStream(new Token[] {t1, t2})));
+ try {
+ iw.addDocument(doc);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+
+ // Document should not be visible:
+ IndexReader r = DirectoryReader.open(iw, true);
+ assertEquals(0, r.numDocs());
+ r.close();
+
+ iw.close();
+ dir.close();
+ }
+
+ public void testMaxPosition() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
+ Document doc = new Document();
+ // This is at position 0:
+ Token t1 = new Token("foo", 0, 3);
+ if (random().nextBoolean()) {
+ t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
+ }
+ Token t2 = new Token("foo", 4, 7);
+ t2.setPositionIncrement(IndexWriter.MAX_POSITION);
+ if (random().nextBoolean()) {
+ t2.setPayload(new BytesRef(new byte[] { 0x1 } ));
+ }
+ doc.add(new TextField("foo", new CannedTokenStream(new Token[] {t1, t2})));
+ iw.addDocument(doc);
+
+ // Document should be visible:
+ IndexReader r = DirectoryReader.open(iw, true);
+ assertEquals(1, r.numDocs());
+ PostingsEnum postings = MultiFields.getTermPositionsEnum(r, null, "foo", new BytesRef("foo"));
+
+ // "foo" appears in docID=0
+ assertEquals(0, postings.nextDoc());
+
+ // "foo" appears 2 times in the doc
+ assertEquals(2, postings.freq());
+
+ // first at pos=0
+ assertEquals(0, postings.nextPosition());
+
+ // next at pos=MAX
+ assertEquals(IndexWriter.MAX_POSITION, postings.nextPosition());
+
+ r.close();
+
+ iw.close();
+ dir.close();
+ }
+}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1673508&r1=1673507&r2=1673508&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Tue Apr 14 18:54:10 2015
@@ -25,10 +25,11 @@ import org.apache.lucene.codecs.FieldsCo
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.index.AssertingLeafReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
@@ -219,6 +220,7 @@ public final class AssertingPostingsForm
for(int i=0;i<freq;i++) {
int pos = postingsEnum.nextPosition();
assert pos >= lastPos: "pos=" + pos + " vs lastPos=" + lastPos + " i=" + i + " freq=" + freq;
+ assert pos <= IndexWriter.MAX_POSITION: "pos=" + pos + " is > IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION;
lastPos = pos;
if (hasOffsets) {