You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/06/10 16:52:17 UTC
svn commit: r1348607 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/search/ lucene/facet/
lucene/facet/src/...
Author: mikemccand
Date: Sun Jun 10 14:52:17 2012
New Revision: 1348607
URL: http://svn.apache.org/viewvc?rev=1348607&view=rev
Log:
LUCENE-4127: don't allow 0 posInc for first token of indexed field
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1348607&r1=1348606&r2=1348607&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Sun Jun 10 14:52:17 2012
@@ -384,6 +384,11 @@ Changes in Runtime Behavior
any of the calls to the Analyzer throw an IOException. QueryParseBase.analyzeRangePart()
will throw a RuntimException if an IOException is thrown by the Analyzer.
+* LUCENE-4127: IndexWriter will now throw IllegalArgumentException if
+ the first token of an indexed field has 0 positionIncrement
+ (previously it silently corrected it to 1, possibly masking bugs).
+ (Robert Muir, Mike McCandless)
+
API Changes
* LUCENE-2302, LUCENE-1458, LUCENE-2111, LUCENE-2514: Terms are no longer
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1348607&r1=1348606&r2=1348607&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Sun Jun 10 14:52:17 2012
@@ -18,9 +18,11 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.IOUtils;
/**
* Holds state for inverting all occurrences of a single
@@ -87,6 +89,8 @@ final class DocInverterPerField extends
// reset the TokenStream to the first token
stream.reset();
+ boolean success2 = false;
+
try {
boolean hasMoreTokens = stream.incrementToken();
@@ -109,8 +113,16 @@ final class DocInverterPerField extends
if (!hasMoreTokens) break;
final int posIncr = posIncrAttribute.getPositionIncrement();
+ if (posIncr < 0) {
+ throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")");
+ }
+ if (fieldState.position == 0 && posIncr == 0) {
+ throw new IllegalArgumentException("first position increment must be > 0 (got 0)");
+ }
int position = fieldState.position + posIncr;
if (position > 0) {
+ // NOTE: confusing: this "mirrors" the
+ // position++ we do below
position--;
} else if (position < 0) {
throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
@@ -147,8 +159,13 @@ final class DocInverterPerField extends
stream.end();
fieldState.offset += offsetAttribute.endOffset();
+ success2 = true;
} finally {
- stream.close();
+ if (!success2) {
+ IOUtils.closeWhileHandlingException(stream);
+ } else {
+ stream.close();
+ }
}
fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1348607&r1=1348606&r2=1348607&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Jun 10 14:52:17 2012
@@ -883,39 +883,16 @@ public class TestIndexWriter extends Luc
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document doc = new Document();
doc.add(new TextField("field", tokens));
- w.addDocument(doc);
- w.commit();
-
- IndexReader r = DirectoryReader.open(dir);
- IndexSearcher s = new IndexSearcher(r);
- PhraseQuery pq = new PhraseQuery();
- pq.add(new Term("field", "a"));
- pq.add(new Term("field", "b"));
- pq.add(new Term("field", "c"));
- ScoreDoc[] hits = s.search(pq, null, 1000).scoreDocs;
- assertEquals(1, hits.length);
-
- Query q = new SpanTermQuery(new Term("field", "a"));
- hits = s.search(q, null, 1000).scoreDocs;
- assertEquals(1, hits.length);
-
- DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(),
- MultiFields.getLiveDocs(s.getIndexReader()),
- "field",
- new BytesRef("a"),
- false);
-
- assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
- assertEquals(1, tps.freq());
- assertEquals(0, tps.nextPosition());
+ try {
+ w.addDocument(doc);
+ fail("did not hit expected exception");
+ } catch (IllegalArgumentException iea) {
+ // expected
+ }
w.close();
-
- r.close();
dir.close();
}
-
-
// LUCENE-1219
public void testBinaryFieldOffsetLength() throws IOException {
Directory dir = newDirectory();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1348607&r1=1348606&r2=1348607&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Sun Jun 10 14:52:17 2012
@@ -61,7 +61,7 @@ public class TestPositionIncrement exten
return new TokenStreamComponents(new Tokenizer(reader) {
// TODO: use CannedTokenStream
private final String[] TOKENS = {"1", "2", "3", "4", "5"};
- private final int[] INCREMENTS = {0, 2, 1, 0, 1};
+ private final int[] INCREMENTS = {1, 2, 1, 0, 1};
private int i = 0;
PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
@@ -222,8 +222,7 @@ public class TestPositionIncrement exten
assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
// "a" occurs 4 times
assertEquals(4, tp.freq());
- int expected = 0;
- assertEquals(expected, tp.nextPosition());
+ assertEquals(0, tp.nextPosition());
assertEquals(1, tp.nextPosition());
assertEquals(3, tp.nextPosition());
assertEquals(6, tp.nextPosition());
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1348607&r1=1348606&r2=1348607&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Sun Jun 10 14:52:17 2012
@@ -546,7 +546,7 @@ public class DirectoryTaxonomyWriter imp
// we write here (e.g., to write parent+2), and need to do a workaround
// in the reader (which knows that anyway only category 0 has a parent
// -1).
- parentStream.set(parent + 1);
+ parentStream.set(Math.max(parent+1, 1));
Document d = new Document();
d.add(parentStreamField);
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java?rev=1348607&r1=1348606&r2=1348607&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java Sun Jun 10 14:52:17 2012
@@ -69,7 +69,7 @@ final class MockPayloadFilter extends To
if (input.incrementToken()) {
payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes()));
int posIncr;
- if (i % 2 == 1) {
+ if (pos == 0 || i % 2 == 1) {
posIncr = 1;
} else {
posIncr = 0;