You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2014/11/12 16:44:36 UTC
svn commit: r1638819 - in /lucene/dev/lucene2878/lucene:
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/index/
core/src/test/org/apache/lucene/search/posfilter/
test-framework/src/java/org/apache/lucene/index/
Author: romseygeek
Date: Wed Nov 12 15:44:36 2014
New Revision: 1638819
URL: http://svn.apache.org/r1638819
Log:
Fix SimpleText omitTF handling
Removed:
lucene/dev/lucene2878/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
Modified:
lucene/dev/lucene2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/lucene2878/lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java
lucene/dev/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
Modified: lucene/dev/lucene2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/lucene2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1638819&r1=1638818&r2=1638819&view=diff
==============================================================================
--- lucene/dev/lucene2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/lucene2878/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Wed Nov 12 15:44:36 2014
@@ -17,6 +17,14 @@ package org.apache.lucene.codecs.simplet
* limitations under the License.
*/
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
@@ -48,14 +56,6 @@ import org.apache.lucene.util.fst.PairOu
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-
import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.DOC;
import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END;
import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END_OFFSET;
@@ -246,6 +246,7 @@ class SimpleTextFieldsReader extends Fie
private final CharsRefBuilder scratchUTF16_2 = new CharsRefBuilder();
private BytesRef payload;
private long nextDocStart;
+ private boolean omitTF;
private boolean readOffsets;
private boolean readPositions;
private int startOffset;
@@ -266,12 +267,14 @@ class SimpleTextFieldsReader extends Fie
this.liveDocs = liveDocs;
nextDocStart = fp;
docID = -1;
+ omitTF = indexOptions == IndexOptions.DOCS;
readPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
readOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
if (!readOffsets) {
startOffset = -1;
endOffset = -1;
}
+ tf = 1;
cost = docFreq;
return this;
}
@@ -291,6 +294,7 @@ class SimpleTextFieldsReader extends Fie
boolean first = true;
in.seek(nextDocStart);
long posStart = 0;
+ int termFreq = 0;
while(true) {
final long lineStart = in.getFilePointer();
SimpleTextUtil.readLine(in, scratch);
@@ -299,16 +303,19 @@ class SimpleTextFieldsReader extends Fie
if (!first && (liveDocs == null || liveDocs.get(docID))) {
nextDocStart = lineStart;
in.seek(posStart);
+ if (!omitTF) {
+ tf = termFreq;
+ }
return docID;
}
scratchUTF16.copyUTF8Bytes(scratch.bytes(), DOC.length, scratch.length()-DOC.length);
docID = ArrayUtil.parseInt(scratchUTF16.chars(), 0, scratchUTF16.length());
- tf = 0;
+ termFreq = 0;
posPending = 0;
first = false;
} else if (StringHelper.startsWith(scratch.get(), FREQ)) {
scratchUTF16.copyUTF8Bytes(scratch.bytes(), FREQ.length, scratch.length()-FREQ.length);
- tf = ArrayUtil.parseInt(scratchUTF16.chars(), 0, scratchUTF16.length());
+ termFreq = ArrayUtil.parseInt(scratchUTF16.chars(), 0, scratchUTF16.length());
posStart = in.getFilePointer();
} else if (StringHelper.startsWith(scratch.get(), POS)) {
// skip
Modified: lucene/dev/lucene2878/lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/lucene2878/lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java?rev=1638819&r1=1638818&r2=1638819&view=diff
==============================================================================
--- lucene/dev/lucene2878/lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java (original)
+++ lucene/dev/lucene2878/lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java Wed Nov 12 15:44:36 2014
@@ -16,10 +16,12 @@ package org.apache.lucene.search.posfilt
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
@@ -28,8 +30,6 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-import java.io.IOException;
-
// We need to store offsets here, so don't use the following Codecs, which don't
// support them.
@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
@@ -37,7 +37,7 @@ public class TestPositionsAndOffsets ext
protected void addDocs(RandomIndexWriter writer) throws IOException {
FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
- fieldType.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
Document doc = new Document();
doc.add(newField(
"field",
Modified: lucene/dev/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1638819&r1=1638818&r2=1638819&view=diff
==============================================================================
--- lucene/dev/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/lucene2878/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Wed Nov 12 15:44:36 2014
@@ -17,8 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
@@ -61,6 +59,8 @@ import org.apache.lucene.util.BytesRefHa
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
+import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
+
/**
* Abstract class to do basic tests for a docvalues format.
* NOTE: This test focuses on the docvalues impl, nothing else.
@@ -2083,7 +2083,7 @@ public abstract class BaseDocValuesForma
);
}
}
-
+
public void testSortedNumericsMultipleValuesVsStoredFields() throws Exception {
assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
int numIterations = atLeast(1);