You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2014/02/14 14:51:27 UTC
svn commit: r1568294 - in /lucene/dev/trunk/lucene: ./
memory/src/java/org/apache/lucene/index/memory/
memory/src/test/org/apache/lucene/index/memory/
test-framework/src/java/org/apache/lucene/analysis/
Author: simonw
Date: Fri Feb 14 13:51:27 2014
New Revision: 1568294
URL: http://svn.apache.org/r1568294
Log:
LUCENE-5444: MemoryIndex did't respect the analyzers offset gap
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1568294&r1=1568293&r2=1568294&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Feb 14 13:51:27 2014
@@ -265,6 +265,10 @@ Bug fixes
* LUCENE-5443: Lucene45DocValuesProducer.ramBytesUsed() may throw
ConcurrentModificationException. (Shai Erera, Simon Willnauer)
+
+* LUCENE-5444: MemoryIndex did't respect the analyzers offset gap and
+ offsets were corrupted if multiple fields with the same name were
+ added to the memory index. (Britta Weber, Simon Willnauer)
API Changes
Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1568294&r1=1568293&r2=1568294&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Feb 14 13:51:27 2014
@@ -295,7 +295,7 @@ public class MemoryIndex {
throw new RuntimeException(ex);
}
- addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName));
+ addField(fieldName, stream, 1.0f, analyzer.getPositionIncrementGap(fieldName), analyzer.getOffsetGap(fieldName));
}
/**
@@ -340,7 +340,7 @@ public class MemoryIndex {
/**
* Equivalent to <code>addField(fieldName, stream, 1.0f)</code>.
- *
+ *
* @param fieldName
* a name to be associated with the text
* @param stream
@@ -370,25 +370,52 @@ public class MemoryIndex {
addField(fieldName, stream, boost, 0);
}
+
/**
* Iterates over the given token stream and adds the resulting terms to the index;
* Equivalent to adding a tokenized, indexed, termVectorStored, unstored,
* Lucene {@link org.apache.lucene.document.Field}.
- * Finally closes the token stream. Note that untokenized keywords can be added with this method via
+ * Finally closes the token stream. Note that untokenized keywords can be added with this method via
* {@link #keywordTokenStream(Collection)}, the Lucene <code>KeywordTokenizer</code> or similar utilities.
- *
+ *
* @param fieldName
* a name to be associated with the text
* @param stream
* the token stream to retrieve tokens from.
* @param boost
* the boost factor for hits for this field
+ *
* @param positionIncrementGap
* the position increment gap if fields with the same name are added more than once
*
+ *
* @see org.apache.lucene.document.Field#setBoost(float)
*/
public void addField(String fieldName, TokenStream stream, float boost, int positionIncrementGap) {
+ addField(fieldName, stream, boost, positionIncrementGap, 1);
+ }
+
+ /**
+ * Iterates over the given token stream and adds the resulting terms to the index;
+ * Equivalent to adding a tokenized, indexed, termVectorStored, unstored,
+ * Lucene {@link org.apache.lucene.document.Field}.
+ * Finally closes the token stream. Note that untokenized keywords can be added with this method via
+ * {@link #keywordTokenStream(Collection)}, the Lucene <code>KeywordTokenizer</code> or similar utilities.
+ *
+ *
+ * @param fieldName
+ * a name to be associated with the text
+ * @param stream
+ * the token stream to retrieve tokens from.
+ * @param boost
+ * the boost factor for hits for this field
+ * @param positionIncrementGap
+ * the position increment gap if fields with the same name are added more than once
+ * @param offsetGap
+ * the offset gap if fields with the same name are added more than once
+ * @see org.apache.lucene.document.Field#setBoost(float)
+ */
+ public void addField(String fieldName, TokenStream stream, float boost, int positionIncrementGap, int offsetGap) {
try {
if (fieldName == null)
throw new IllegalArgumentException("fieldName must not be null");
@@ -403,10 +430,12 @@ public class MemoryIndex {
final SliceByteStartArray sliceArray;
Info info = null;
long sumTotalTermFreq = 0;
+ int offset = 0;
if ((info = fields.get(fieldName)) != null) {
numTokens = info.numTokens;
numOverlapTokens = info.numOverlapTokens;
pos = info.lastPosition + positionIncrementGap;
+ offset = info.lastOffset + offsetGap;
terms = info.terms;
boost *= info.boost;
sliceArray = info.sliceArray;
@@ -447,8 +476,8 @@ public class MemoryIndex {
postingsWriter.writeInt(pos);
} else {
postingsWriter.writeInt(pos);
- postingsWriter.writeInt(offsetAtt.startOffset());
- postingsWriter.writeInt(offsetAtt.endOffset());
+ postingsWriter.writeInt(offsetAtt.startOffset() + offset);
+ postingsWriter.writeInt(offsetAtt.endOffset() + offset);
}
sliceArray.end[ord] = postingsWriter.getCurrentOffset();
}
@@ -456,7 +485,7 @@ public class MemoryIndex {
// ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
if (numTokens > 0) {
- fields.put(fieldName, new Info(terms, sliceArray, numTokens, numOverlapTokens, boost, pos, sumTotalTermFreq));
+ fields.put(fieldName, new Info(terms, sliceArray, numTokens, numOverlapTokens, boost, pos, offsetAtt.endOffset() + offset, sumTotalTermFreq));
sortedFields = null; // invalidate sorted view, if any
}
} catch (Exception e) { // can never happen
@@ -670,7 +699,10 @@ public class MemoryIndex {
/** the last position encountered in this field for multi field support*/
private int lastPosition;
- public Info(BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, long sumTotalTermFreq) {
+ /** the last offset encountered in this field for multi field support*/
+ private int lastOffset;
+
+ public Info(BytesRefHash terms, SliceByteStartArray sliceArray, int numTokens, int numOverlapTokens, float boost, int lastPosition, int lastOffset, long sumTotalTermFreq) {
this.terms = terms;
this.sliceArray = sliceArray;
this.numTokens = numTokens;
@@ -678,6 +710,7 @@ public class MemoryIndex {
this.boost = boost;
this.sumTotalTermFreq = sumTotalTermFreq;
this.lastPosition = lastPosition;
+ this.lastOffset = lastOffset;
}
public long getSumTotalTermFreq() {
Modified: lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1568294&r1=1568293&r2=1568294&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Fri Feb 14 13:51:27 2014
@@ -21,7 +21,6 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.Reader;
import java.util.HashSet;
import java.util.Set;
@@ -39,6 +38,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.DirectoryReader;
@@ -49,7 +50,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
@@ -76,6 +76,8 @@ import org.apache.lucene.util.RamUsageEs
import org.apache.lucene.util.RecyclingByteBlockAllocator;
import org.apache.lucene.util._TestUtil;
+import static org.hamcrest.CoreMatchers.equalTo;
+
/**
* Verifies that Lucene MemoryIndex and RAMDirectory have the same behaviour,
* returning the same results for queries on some randomish indexes.
@@ -469,4 +471,80 @@ public class MemoryIndexTest extends Bas
TopDocs docs = searcher.search(new TermQuery(new Term("foo", "")), 10);
assertEquals(1, docs.totalHits);
}
+
+ public void testDuelMemoryIndexCoreDirectoryWithArrayField() throws Exception {
+
+ final String field_name = "text";
+ MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
+ if (random().nextBoolean()) {
+ mockAnalyzer.setOffsetGap(random().nextInt(100));
+ }
+ //index into a random directory
+ FieldType type = new FieldType(TextField.TYPE_STORED);
+ type.setStoreTermVectorOffsets(true);
+ type.setStoreTermVectorPayloads(false);
+ type.setStoreTermVectorPositions(true);
+ type.setStoreTermVectors(true);
+ type.freeze();
+
+ Document doc = new Document();
+ doc.add(new Field(field_name, "la la", type));
+ doc.add(new Field(field_name, "foo bar foo bar foo", type));
+
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), TEST_VERSION_CURRENT, mockAnalyzer));
+ writer.updateDocument(new Term("id", "1"), doc);
+ writer.commit();
+ writer.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+
+ //Index document in Memory index
+ MemoryIndex memIndex = new MemoryIndex(true);
+ memIndex.addField(field_name, "la la", mockAnalyzer);
+ memIndex.addField(field_name, "foo bar foo bar foo", mockAnalyzer);
+
+ //compare term vectors
+ Terms ramTv = reader.getTermVector(0, field_name);
+ IndexReader memIndexReader = memIndex.createSearcher().getIndexReader();
+ Terms memTv = memIndexReader.getTermVector(0, field_name);
+
+ compareTermVectors(ramTv, memTv, field_name);
+ memIndexReader.close();
+ reader.close();
+ dir.close();
+
+ }
+
+ protected void compareTermVectors(Terms terms, Terms memTerms, String field_name) throws IOException {
+
+ TermsEnum termEnum = terms.iterator(null);
+ TermsEnum memTermEnum = memTerms.iterator(null);
+
+ while (termEnum.next() != null) {
+ assertNotNull(memTermEnum.next());
+ assertThat(termEnum.totalTermFreq(), equalTo(memTermEnum.totalTermFreq()));
+
+ DocsAndPositionsEnum docsPosEnum = termEnum.docsAndPositions(null, null, 0);
+ DocsAndPositionsEnum memDocsPosEnum = memTermEnum.docsAndPositions(null, null, 0);
+ String currentTerm = termEnum.term().utf8ToString();
+
+ assertThat("Token mismatch for field: " + field_name, currentTerm, equalTo(memTermEnum.term().utf8ToString()));
+
+ docsPosEnum.nextDoc();
+ memDocsPosEnum.nextDoc();
+
+ int freq = docsPosEnum.freq();
+ assertThat(freq, equalTo(memDocsPosEnum.freq()));
+ for (int i = 0; i < freq; i++) {
+ String failDesc = " (field:" + field_name + " term:" + currentTerm + ")";
+ int memPos = memDocsPosEnum.nextPosition();
+ int pos = docsPosEnum.nextPosition();
+ assertThat("Position test failed" + failDesc, memPos, equalTo(pos));
+ assertThat("Start offset test failed" + failDesc, memDocsPosEnum.startOffset(), equalTo(docsPosEnum.startOffset()));
+ assertThat("End offset test failed" + failDesc, memDocsPosEnum.endOffset(), equalTo(docsPosEnum.endOffset()));
+ assertThat("Missing payload test failed" + failDesc, docsPosEnum.getPayload(), equalTo(null));
+ }
+ }
+ assertNull("Still some tokens not processed", memTermEnum.next());
+ }
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java?rev=1568294&r1=1568293&r2=1568294&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java Fri Feb 14 13:51:27 2014
@@ -46,6 +46,7 @@ public final class MockAnalyzer extends
private final boolean lowerCase;
private final CharacterRunAutomaton filter;
private int positionIncrementGap;
+ private Integer offsetGap;
private final Random random;
private Map<String,Integer> previousMappings = new HashMap<String,Integer>();
private boolean enableChecks = true;
@@ -134,6 +135,23 @@ public final class MockAnalyzer extends
public int getPositionIncrementGap(String fieldName){
return positionIncrementGap;
}
+
+ /**
+ * Set a new offset gap which will then be added to the offset when several fields with the same name are indexed
+ * @param offsetGap The offset gap that should be used.
+ */
+ public void setOffsetGap(int offsetGap){
+ this.offsetGap = offsetGap;
+ }
+
+ /**
+ * Get the offset gap between tokens in fields if several fields with the same name were added.
+ * @param fieldName Currently not used, the same offset gap is returned for each field.
+ */
+ @Override
+ public int getOffsetGap(String fieldName){
+ return offsetGap == null ? super.getOffsetGap(fieldName) : offsetGap;
+ }
/**
* Toggle consumer workflow checking: if your test consumes tokenstreams normally you