You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/08/30 17:06:49 UTC
svn commit: r1518989 [6/10] - in /lucene/dev/branches/lucene3069: ./
dev-tools/ dev-tools/idea/solr/contrib/velocity/ dev-tools/maven/
dev-tools/maven/solr/core/src/java/ lucene/ lucene/analysis/
lucene/analysis/common/ lucene/analysis/common/src/java/...
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java Fri Aug 30 15:06:42 2013
@@ -47,6 +47,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class FastVectorHighlighterTest extends LuceneTestCase {
@@ -298,6 +299,49 @@ public class FastVectorHighlighterTest e
writer.close();
dir.close();
}
+
+ public void testLotsOfPhrases() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
+ FieldType type = new FieldType(TextField.TYPE_STORED);
+ type.setStoreTermVectorOffsets(true);
+ type.setStoreTermVectorPositions(true);
+ type.setStoreTermVectors(true);
+ type.freeze();
+ String[] terms = { "org", "apache", "lucene"};
+ int iters = 1000; // don't let it go too big, or jenkins will stack overflow: atLeast(1000);
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < iters; i++) {
+ builder.append(terms[random().nextInt(terms.length)]).append(" ");
+ if (random().nextInt(6) == 3) {
+ builder.append("solr").append(" ");
+ }
+ }
+ Document doc = new Document();
+ Field field = new Field("field", builder.toString(), type);
+ doc.add(field);
+ writer.addDocument(doc);
+ PhraseQuery query = new PhraseQuery();
+ query.add(new Term("field", "org"));
+ query.add(new Term("field", "apache"));
+ query.add(new Term("field", "lucene"));
+
+
+ FastVectorHighlighter highlighter = new FastVectorHighlighter();
+ IndexReader reader = DirectoryReader.open(writer, true);
+ IndexSearcher searcher = newSearcher(reader);
+ TopDocs hits = searcher.search(query, 10);
+ assertEquals(1, hits.totalHits);
+ FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);
+ String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
+ for (int i = 0; i < bestFragments.length; i++) {
+ String result = bestFragments[i].replaceAll("<b>org apache lucene</b>", "FOOBAR");
+ assertFalse(result.contains("org apache lucene"));
+ }
+ reader.close();
+ writer.close();
+ dir.close();
+ }
public void testOverlappingPhrases() throws IOException {
final Analyzer analyzer = new Analyzer() {
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Fri Aug 30 15:06:42 2013
@@ -85,7 +85,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] preTags = { "[" };
String[] postTags = { "]" };
- assertEquals( "<h1> [a] </h1>",
+ assertEquals( "<h1> [a] </h1>",
sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );
}
Modified: lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Aug 30 15:06:42 2013
@@ -756,6 +756,11 @@ public class MemoryIndex {
return null;
}
+ @Override
+ public Bits getDocsWithField(String field) throws IOException {
+ return null;
+ }
+
private class MemoryFields extends Fields {
@Override
public Iterator<String> iterator() {
Modified: lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Fri Aug 30 15:06:42 2013
@@ -176,7 +176,7 @@ public class MemoryIndexTest extends Bas
private void duellReaders(CompositeReader other, AtomicReader memIndexReader)
throws IOException {
- AtomicReader competitor = new SlowCompositeReaderWrapper(other);
+ AtomicReader competitor = SlowCompositeReaderWrapper.wrap(other);
Fields memFields = memIndexReader.fields();
for (String field : competitor.fields()) {
Terms memTerms = memFields.terms(field);
Modified: lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java Fri Aug 30 15:06:42 2013
@@ -221,6 +221,27 @@ public class SortingAtomicReader extends
}
}
+ private static class SortingBits implements Bits {
+
+ private final Bits in;
+ private final Sorter.DocMap docMap;
+
+ public SortingBits(final Bits in, Sorter.DocMap docMap) {
+ this.in = in;
+ this.docMap = docMap;
+ }
+
+ @Override
+ public boolean get(int index) {
+ return in.get(docMap.newToOld(index));
+ }
+
+ @Override
+ public int length() {
+ return in.length();
+ }
+ }
+
private static class SortingSortedDocValues extends SortedDocValues {
private final SortedDocValues in;
@@ -743,20 +764,9 @@ public class SortingAtomicReader extends
final Bits inLiveDocs = in.getLiveDocs();
if (inLiveDocs == null) {
return null;
+ } else {
+ return new SortingBits(inLiveDocs, docMap);
}
- return new Bits() {
-
- @Override
- public boolean get(int index) {
- return inLiveDocs.get(docMap.newToOld(index));
- }
-
- @Override
- public int length() {
- return inLiveDocs.length();
- }
-
- };
}
@Override
@@ -797,6 +807,16 @@ public class SortingAtomicReader extends
}
@Override
+ public Bits getDocsWithField(String field) throws IOException {
+ Bits bits = in.getDocsWithField(field);
+ if (bits == null || bits instanceof Bits.MatchAllBits || bits instanceof Bits.MatchNoBits) {
+ return bits;
+ } else {
+ return new SortingBits(bits, docMap);
+ }
+ }
+
+ @Override
public Fields getTermVectors(final int docID) throws IOException {
return in.getTermVectors(docMap.newToOld(docID));
}
Modified: lucene/dev/branches/lucene3069/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (original)
+++ lucene/dev/branches/lucene3069/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Fri Aug 30 15:06:42 2013
@@ -114,7 +114,6 @@ public abstract class SorterTestBase ext
public PositionsTokenStream() {
term = addAttribute(CharTermAttribute.class);
- term.append(DOC_POSITIONS_TERM);
payload = addAttribute(PayloadAttribute.class);
offset = addAttribute(OffsetAttribute.class);
}
@@ -125,6 +124,8 @@ public abstract class SorterTestBase ext
return false;
}
+ clearAttributes();
+ term.append(DOC_POSITIONS_TERM);
payload.setPayload(new BytesRef(Integer.toString(pos)));
offset.setOffset(off, off);
--pos;
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java Fri Aug 30 15:06:42 2013
@@ -45,12 +45,13 @@ public class BytesRefFieldSource extends
// To be sorted or not to be sorted, that is the question
// TODO: do it cleaner?
if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+ final Bits docsWithField = FieldCache.DEFAULT.getDocsWithField(readerContext.reader(), field);
final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field);
return new FunctionValues() {
@Override
public boolean exists(int doc) {
- return true; // doc values are dense
+ return docsWithField.get(doc);
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java Fri Aug 30 15:06:42 2013
@@ -68,7 +68,7 @@ public class DoubleFieldSource extends F
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -142,7 +142,7 @@ public class DoubleFieldSource extends F
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java Fri Aug 30 15:06:42 2013
@@ -72,7 +72,7 @@ public class FloatFieldSource extends Fi
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -88,7 +88,7 @@ public class FloatFieldSource extends Fi
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java Fri Aug 30 15:06:42 2013
@@ -95,7 +95,7 @@ public class IntFieldSource extends Fiel
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -150,7 +150,7 @@ public class IntFieldSource extends Fiel
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java Fri Aug 30 15:06:42 2013
@@ -81,7 +81,7 @@ public class LongFieldSource extends Fie
@Override
public boolean exists(int doc) {
- return valid.get(doc);
+ return arr.get(doc) != 0 || valid.get(doc);
}
@Override
@@ -141,7 +141,7 @@ public class LongFieldSource extends Fie
@Override
public void fillValue(int doc) {
mval.value = arr.get(doc);
- mval.exists = valid.get(doc);
+ mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java Fri Aug 30 15:06:42 2013
@@ -70,9 +70,7 @@ public class OrdFieldSource extends Valu
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final int off = readerContext.docBase;
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
- final AtomicReader r = topReader instanceof CompositeReader
- ? new SlowCompositeReaderWrapper((CompositeReader)topReader)
- : (AtomicReader) topReader;
+ final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
return new IntDocValues(this) {
protected String toTerm(String readableValue) {
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java Fri Aug 30 15:06:42 2013
@@ -69,9 +69,7 @@ public class ReverseOrdFieldSource exten
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
- final AtomicReader r = topReader instanceof CompositeReader
- ? new SlowCompositeReaderWrapper((CompositeReader)topReader)
- : (AtomicReader) topReader;
+ final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
final int off = readerContext.docBase;
final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java Fri Aug 30 15:06:42 2013
@@ -56,7 +56,7 @@ public class BooleanFilterTest extends L
addDoc(writer, "guest", "020", "20050101", "Y");
addDoc(writer, "admin", "020", "20050101", "Maybe");
addDoc(writer, "admin guest", "030", "20050101", "N");
- reader = new SlowCompositeReaderWrapper(writer.getReader());
+ reader = SlowCompositeReaderWrapper.wrap(writer.getReader());
writer.close();
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java Fri Aug 30 15:06:42 2013
@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
@@ -311,7 +312,7 @@ public class CommonTermsQueryTest extend
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
createRandomIndex(atLeast(50), w, random().nextLong());
DirectoryReader reader = w.getReader();
- SlowCompositeReaderWrapper wrapper = new SlowCompositeReaderWrapper(reader);
+ AtomicReader wrapper = SlowCompositeReaderWrapper.wrap(reader);
String field = "body";
Terms terms = wrapper.terms(field);
PriorityQueue<TermAndFreq> lowFreqQueue = new PriorityQueue<CommonTermsQueryTest.TermAndFreq>(
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermFilterTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermFilterTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermFilterTest.java Fri Aug 30 15:06:42 2013
@@ -59,7 +59,7 @@ public class TermFilterTest extends Luce
Document doc = new Document();
doc.add(newStringField(fieldName, "value1", Field.Store.NO));
w.addDocument(doc);
- IndexReader reader = new SlowCompositeReaderWrapper(w.getReader());
+ IndexReader reader = SlowCompositeReaderWrapper.wrap(w.getReader());
assertTrue(reader.getContext() instanceof AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
w.close();
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java Fri Aug 30 15:06:42 2013
@@ -74,7 +74,7 @@ public class TermsFilterTest extends Luc
doc.add(newStringField(fieldName, "" + term, Field.Store.YES));
w.addDocument(doc);
}
- IndexReader reader = new SlowCompositeReaderWrapper(w.getReader());
+ IndexReader reader = SlowCompositeReaderWrapper.wrap(w.getReader());
assertTrue(reader.getContext() instanceof AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
w.close();
Modified: lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java Fri Aug 30 15:06:42 2013
@@ -311,7 +311,7 @@ public class TestMultiFieldQueryParser e
MockAnalyzer stdAnalyzer = new MockAnalyzer(random());
public AnalyzerReturningNull() {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java Fri Aug 30 15:06:42 2013
@@ -347,7 +347,7 @@ public class TestMultiFieldQPHelper exte
MockAnalyzer stdAnalyzer = new MockAnalyzer(random());
public AnalyzerReturningNull() {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java Fri Aug 30 15:06:42 2013
@@ -64,7 +64,7 @@ public class TestNumericRangeFilterBuild
IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
writer.commit();
try {
- AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(ramDir));
+ AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(ramDir));
try {
assertNull(filter.getDocIdSet(reader.getContext(), reader.getLiveDocs()));
}
Modified: lucene/dev/branches/lucene3069/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/site/xsl/index.xsl?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/lucene3069/lucene/site/xsl/index.xsl Fri Aug 30 15:06:42 2013
@@ -75,7 +75,7 @@
<li><a href="SYSTEM_REQUIREMENTS.html">System Requirements</a>: Minimum and supported Java versions.</li>
<li><a href="MIGRATE.html">Migration Guide</a>: What changed in Lucene 4; how to migrate code from Lucene 3.x.</li>
<li><a href="JRE_VERSION_MIGRATION.html">JRE Version Migration</a>: Information about upgrading between major JRE versions.</li>
- <li><a href="core/org/apache/lucene/codecs/lucene42/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
+ <li><a href="core/org/apache/lucene/codecs/lucene45/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
<li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
<li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
<li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>
Modified: lucene/dev/branches/lucene3069/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java Fri Aug 30 15:06:42 2013
@@ -78,10 +78,12 @@ public class BBoxSimilarityValueSource e
@Override
public float floatVal(int doc) {
+ double minXVal = minX.get(doc);
+ double maxXVal = maxX.get(doc);
// make sure it has minX and area
- if (validMinX.get(doc) && validMaxX.get(doc)) {
+ if ((minXVal != 0 || validMinX.get(doc)) && (maxXVal != 0 || validMaxX.get(doc))) {
rect.reset(
- minX.get(doc), maxX.get(doc),
+ minXVal, maxXVal,
minY.get(doc), maxY.get(doc));
return (float) similarity.score(rect, null);
} else {
Modified: lucene/dev/branches/lucene3069/lucene/suggest/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/suggest/build.xml?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/suggest/build.xml (original)
+++ lucene/dev/branches/lucene3069/lucene/suggest/build.xml Fri Aug 30 15:06:42 2013
@@ -34,6 +34,15 @@
<path refid="base.classpath"/>
</path>
+
+ <target name="javadocs" depends="javadocs-queries,compile-core">
+ <invoke-module-javadoc>
+ <links>
+ <link href="../analyzers-common"/>
+ </links>
+ </invoke-module-javadoc>
+ </target>
+
<target name="compile-core" depends="jar-misc, jar-analyzers-common, common.compile-core" />
</project>
Modified: lucene/dev/branches/lucene3069/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/lucene3069/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Fri Aug 30 15:06:42 2013
@@ -34,7 +34,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -99,10 +99,10 @@ public class AnalyzingInfixSuggester ext
protected final static String TEXT_FIELD_NAME = "text";
private final Analyzer queryAnalyzer;
- private final Analyzer indexAnalyzer;
- private final Version matchVersion;
+ final Analyzer indexAnalyzer;
+ final Version matchVersion;
private final File indexPath;
- private final int minPrefixChars;
+ final int minPrefixChars;
private Directory dir;
/** {@link IndexSearcher} used for lookups. */
@@ -161,7 +161,7 @@ public class AnalyzingInfixSuggester ext
* codec to use. */
protected IndexWriterConfig getIndexWriterConfig(Version matchVersion, Analyzer indexAnalyzer) {
IndexWriterConfig iwc = new IndexWriterConfig(matchVersion, indexAnalyzer);
- iwc.setCodec(new Lucene42Codec());
+ iwc.setCodec(new Lucene45Codec());
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
return iwc;
}
@@ -193,7 +193,7 @@ public class AnalyzingInfixSuggester ext
AtomicReader r = null;
boolean success = false;
try {
- Analyzer gramAnalyzer = new AnalyzerWrapper() {
+ Analyzer gramAnalyzer = new AnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
return indexAnalyzer;
@@ -254,7 +254,7 @@ public class AnalyzingInfixSuggester ext
}
//System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");
- r = new SlowCompositeReaderWrapper(DirectoryReader.open(w, false));
+ r = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(w, false));
//long t1 = System.nanoTime();
w.rollback();
Modified: lucene/dev/branches/lucene3069/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Fri Aug 30 15:06:42 2013
@@ -25,11 +25,8 @@ import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PrefixQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.TermFreqPayload;
import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
@@ -294,64 +291,39 @@ public class AnalyzingInfixSuggesterTest
suggester.close();
}
- public void testForkLastToken() throws Exception {
- Analyzer a = new Analyzer() {
+ public void testSuggestStopFilter() throws Exception {
+ final CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "a");
+ Analyzer indexAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokens = new MockTokenizer(reader);
- // ForkLastTokenFilter is a bit evil:
- tokens.setEnableChecks(false);
return new TokenStreamComponents(tokens,
- new StopKeywordFilter(TEST_VERSION_CURRENT,
- new ForkLastTokenFilter(tokens), StopKeywordFilter.makeStopSet(TEST_VERSION_CURRENT, "a")));
+ new StopFilter(TEST_VERSION_CURRENT, tokens, stopWords));
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a bob for apples", 10, new BytesRef("foobaz")),
- };
-
- File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
-
- AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+ Analyzer queryAnalyzer = new Analyzer() {
@Override
- protected Query finishQuery(BooleanQuery in, boolean allTermsRequired) {
- List<BooleanClause> clauses = in.clauses();
- if (clauses.size() >= 2 && allTermsRequired) {
- String t1 = getTerm(clauses.get(clauses.size()-2).getQuery());
- String t2 = getTerm(clauses.get(clauses.size()-1).getQuery());
- if (t1.equals(t2)) {
- // The last 2 tokens came from
- // ForkLastTokenFilter; we remove them and
- // replace them with a MUST BooleanQuery that
- // SHOULDs the two of them together:
- BooleanQuery sub = new BooleanQuery();
- BooleanClause other = clauses.get(clauses.size()-2);
- sub.add(new BooleanClause(clauses.get(clauses.size()-2).getQuery(), BooleanClause.Occur.SHOULD));
- sub.add(new BooleanClause(clauses.get(clauses.size()-1).getQuery(), BooleanClause.Occur.SHOULD));
- clauses.subList(clauses.size()-2, clauses.size()).clear();
- clauses.add(new BooleanClause(sub, BooleanClause.Occur.MUST));
- }
- }
- return in;
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ MockTokenizer tokens = new MockTokenizer(reader);
+ return new TokenStreamComponents(tokens,
+ new SuggestStopFilter(tokens, stopWords));
}
+ };
- private String getTerm(Query query) {
- if (query instanceof TermQuery) {
- return ((TermQuery) query).getTerm().text();
- } else if (query instanceof PrefixQuery) {
- return ((PrefixQuery) query).getPrefix().text();
- } else {
- return null;
- }
- }
+ File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+ AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, indexAnalyzer, queryAnalyzer, 3) {
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("a bob for apples", 10, new BytesRef("foobaz")),
+ };
+
suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("a", random()), 10, true, true);
assertEquals(1, results.size());
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Fri Aug 30 15:06:42 2013
@@ -111,7 +111,8 @@ public abstract class BaseTokenStreamTes
// arriving to pos Y have the same endOffset)
// - offsets only move forwards (startOffset >=
// lastStartOffset)
- public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], Integer finalOffset,
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[],
+ int posLengths[], Integer finalOffset, Integer finalPosInc, boolean[] keywordAtts,
boolean offsetsAreCorrect) throws IOException {
assertNotNull(output);
CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class);
@@ -135,7 +136,7 @@ public abstract class BaseTokenStreamTes
}
PositionIncrementAttribute posIncrAtt = null;
- if (posIncrements != null) {
+ if (posIncrements != null || finalPosInc != null) {
assertTrue("has no PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class));
posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class);
}
@@ -145,6 +146,12 @@ public abstract class BaseTokenStreamTes
assertTrue("has no PositionLengthAttribute", ts.hasAttribute(PositionLengthAttribute.class));
posLengthAtt = ts.getAttribute(PositionLengthAttribute.class);
}
+
+ KeywordAttribute keywordAtt = null;
+ if (keywordAtts != null) {
+ assertTrue("has no KeywordAttribute", ts.hasAttribute(KeywordAttribute.class));
+ keywordAtt = ts.getAttribute(KeywordAttribute.class);
+ }
// Maps position to the start/end offset:
final Map<Integer,Integer> posToStartOffset = new HashMap<Integer,Integer>();
@@ -161,22 +168,31 @@ public abstract class BaseTokenStreamTes
if (typeAtt != null) typeAtt.setType("bogusType");
if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657);
if (posLengthAtt != null) posLengthAtt.setPositionLength(45987653);
+ if (keywordAtt != null) keywordAtt.setKeyword((i&1) == 0);
checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before
assertTrue("token "+i+" does not exist", ts.incrementToken());
assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled());
assertEquals("term "+i, output[i], termAtt.toString());
- if (startOffsets != null)
+ if (startOffsets != null) {
assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset());
- if (endOffsets != null)
+ }
+ if (endOffsets != null) {
assertEquals("endOffset "+i, endOffsets[i], offsetAtt.endOffset());
- if (types != null)
+ }
+ if (types != null) {
assertEquals("type "+i, types[i], typeAtt.type());
- if (posIncrements != null)
+ }
+ if (posIncrements != null) {
assertEquals("posIncrement "+i, posIncrements[i], posIncrAtt.getPositionIncrement());
- if (posLengths != null)
+ }
+ if (posLengths != null) {
assertEquals("posLength "+i, posLengths[i], posLengthAtt.getPositionLength());
+ }
+ if (keywordAtts != null) {
+ assertEquals("keywordAtt " + i, keywordAtts[i], keywordAtt.isKeyword());
+ }
// we can enforce some basic things about a few attributes even if the caller doesn't check:
if (offsetAtt != null) {
@@ -239,17 +255,47 @@ public abstract class BaseTokenStreamTes
assertTrue("posLength must be >= 1", posLengthAtt.getPositionLength() >= 1);
}
}
- assertFalse("TokenStream has more tokens than expected (expected count=" + output.length + ")", ts.incrementToken());
+
+ if (ts.incrementToken()) {
+ fail("TokenStream has more tokens than expected (expected count=" + output.length + "); extra token=" + termAtt.toString());
+ }
+
+ // repeat our extra safety checks for end()
+ ts.clearAttributes();
+ if (termAtt != null) termAtt.setEmpty().append("bogusTerm");
+ if (offsetAtt != null) offsetAtt.setOffset(14584724,24683243);
+ if (typeAtt != null) typeAtt.setType("bogusType");
+ if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657);
+ if (posLengthAtt != null) posLengthAtt.setPositionLength(45987653);
+
+ checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before
+
ts.end();
+ assertTrue("super.end()/clearAttributes() was not called correctly in end()", checkClearAtt.getAndResetClearCalled());
+
if (finalOffset != null) {
- assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset());
+ assertEquals("finalOffset", finalOffset.intValue(), offsetAtt.endOffset());
}
if (offsetAtt != null) {
assertTrue("finalOffset must be >= 0", offsetAtt.endOffset() >= 0);
}
+ if (finalPosInc != null) {
+ assertEquals("finalPosInc", finalPosInc.intValue(), posIncrAtt.getPositionIncrement());
+ }
+
ts.close();
}
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[],
+ int posLengths[], Integer finalOffset, boolean[] keywordAtts,
+ boolean offsetsAreCorrect) throws IOException {
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, posLengths, finalOffset, null, null, offsetsAreCorrect);
+ }
+
+ public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], Integer finalOffset, boolean offsetsAreCorrect) throws IOException {
+ assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, posLengths, finalOffset, null, offsetsAreCorrect);
+ }
+
public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], Integer finalOffset) throws IOException {
assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, posLengths, finalOffset, true);
}
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java Fri Aug 30 15:06:42 2013
@@ -60,7 +60,7 @@ public final class MockAnalyzer extends
* @param filter DFA describing how terms should be filtered (set of stopwords, etc)
*/
public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter) {
- super(new PerFieldReuseStrategy());
+ super(PER_FIELD_REUSE_STRATEGY);
// TODO: this should be solved in a different way; Random should not be shared (!).
this.random = new Random(random.nextLong());
this.runAutomaton = runAutomaton;
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java Fri Aug 30 15:06:42 2013
@@ -58,7 +58,8 @@ public final class MockTokenFilter exten
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-
+ private int skippedPositions;
+
/**
* Create a new MockTokenFilter.
*
@@ -76,7 +77,7 @@ public final class MockTokenFilter exten
// initial token with posInc=0 ever
// return the first non-stop word found
- int skippedPositions = 0;
+ skippedPositions = 0;
while (input.incrementToken()) {
if (!filter.run(termAtt.buffer(), 0, termAtt.length())) {
posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
@@ -87,4 +88,16 @@ public final class MockTokenFilter exten
// reached EOS -- return false
return false;
}
+
+ @Override
+ public void end() throws IOException {
+ super.end();
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ skippedPositions = 0;
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java Fri Aug 30 15:06:42 2013
@@ -244,6 +244,7 @@ public class MockTokenizer extends Token
@Override
public void end() throws IOException {
+ super.end();
int finalOffset = correctOffset(off);
offsetAtt.setOffset(finalOffset, finalOffset);
// some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false.
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java Fri Aug 30 15:06:42 2013
@@ -23,10 +23,10 @@ import org.apache.lucene.codecs.NormsFor
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
/**
- * Acts like {@link Lucene42Codec} but with additional asserts.
+ * Acts like {@link Lucene45Codec} but with additional asserts.
*/
public final class AssertingCodec extends FilterCodec {
@@ -37,7 +37,7 @@ public final class AssertingCodec extend
private final NormsFormat norms = new AssertingNormsFormat();
public AssertingCodec() {
- super("Asserting", new Lucene42Codec());
+ super("Asserting", new Lucene45Codec());
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Fri Aug 30 15:06:42 2013
@@ -24,7 +24,7 @@ import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
+import org.apache.lucene.codecs.lucene45.Lucene45DocValuesFormat;
import org.apache.lucene.index.AssertingAtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
@@ -33,15 +33,16 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.OpenBitSet;
/**
- * Just like {@link Lucene42DocValuesFormat} but with additional asserts.
+ * Just like {@link Lucene45DocValuesFormat} but with additional asserts.
*/
public class AssertingDocValuesFormat extends DocValuesFormat {
- private final DocValuesFormat in = new Lucene42DocValuesFormat();
+ private final DocValuesFormat in = new Lucene45DocValuesFormat();
public AssertingDocValuesFormat() {
super("Asserting");
@@ -75,11 +76,10 @@ public class AssertingDocValuesFormat ex
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
int count = 0;
for (Number v : values) {
- assert v != null;
count++;
}
assert count == maxDoc;
- checkIterator(values.iterator(), maxDoc);
+ checkIterator(values.iterator(), maxDoc, true);
in.addNumericField(field, values);
}
@@ -87,12 +87,11 @@ public class AssertingDocValuesFormat ex
public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
int count = 0;
for (BytesRef b : values) {
- assert b != null;
- assert b.isValid();
+ assert b == null || b.isValid();
count++;
}
assert count == maxDoc;
- checkIterator(values.iterator(), maxDoc);
+ checkIterator(values.iterator(), maxDoc, true);
in.addBinaryField(field, values);
}
@@ -117,15 +116,17 @@ public class AssertingDocValuesFormat ex
for (Number v : docToOrd) {
assert v != null;
int ord = v.intValue();
- assert ord >= 0 && ord < valueCount;
- seenOrds.set(ord);
+ assert ord >= -1 && ord < valueCount;
+ if (ord >= 0) {
+ seenOrds.set(ord);
+ }
count++;
}
assert count == maxDoc;
assert seenOrds.cardinality() == valueCount;
- checkIterator(values.iterator(), valueCount);
- checkIterator(docToOrd.iterator(), maxDoc);
+ checkIterator(values.iterator(), valueCount, false);
+ checkIterator(docToOrd.iterator(), maxDoc, false);
in.addSortedField(field, values, docToOrd);
}
@@ -169,38 +170,80 @@ public class AssertingDocValuesFormat ex
assert docCount == maxDoc;
assert seenOrds.cardinality() == valueCount;
- checkIterator(values.iterator(), valueCount);
- checkIterator(docToOrdCount.iterator(), maxDoc);
- checkIterator(ords.iterator(), ordCount);
+ checkIterator(values.iterator(), valueCount, false);
+ checkIterator(docToOrdCount.iterator(), maxDoc, false);
+ checkIterator(ords.iterator(), ordCount, false);
in.addSortedSetField(field, values, docToOrdCount, ords);
}
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+ }
+
+ static class AssertingNormsConsumer extends DocValuesConsumer {
+ private final DocValuesConsumer in;
+ private final int maxDoc;
+
+ AssertingNormsConsumer(DocValuesConsumer in, int maxDoc) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ }
- private <T> void checkIterator(Iterator<T> iterator, long expectedSize) {
- for (long i = 0; i < expectedSize; i++) {
- boolean hasNext = iterator.hasNext();
- assert hasNext;
- T v = iterator.next();
+ @Override
+ public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
+ int count = 0;
+ for (Number v : values) {
assert v != null;
- try {
- iterator.remove();
- throw new AssertionError("broken iterator (supports remove): " + iterator);
- } catch (UnsupportedOperationException expected) {
- // ok
- }
- }
- assert !iterator.hasNext();
- try {
- iterator.next();
- throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
- } catch (NoSuchElementException expected) {
- // ok
+ count++;
}
+ assert count == maxDoc;
+ checkIterator(values.iterator(), maxDoc, false);
+ in.addNumericField(field, values);
}
-
+
@Override
public void close() throws IOException {
in.close();
}
+
+ @Override
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static <T> void checkIterator(Iterator<T> iterator, long expectedSize, boolean allowNull) {
+ for (long i = 0; i < expectedSize; i++) {
+ boolean hasNext = iterator.hasNext();
+ assert hasNext;
+ T v = iterator.next();
+ assert allowNull || v != null;
+ try {
+ iterator.remove();
+ throw new AssertionError("broken iterator (supports remove): " + iterator);
+ } catch (UnsupportedOperationException expected) {
+ // ok
+ }
+ }
+ assert !iterator.hasNext();
+ try {
+ iterator.next();
+ throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
+ } catch (NoSuchElementException expected) {
+ // ok
+ }
}
static class AssertingDocValuesProducer extends DocValuesProducer {
@@ -244,6 +287,15 @@ public class AssertingDocValuesFormat ex
assert values != null;
return new AssertingAtomicReader.AssertingSortedSetDocValues(values, maxDoc);
}
+
+ @Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ assert field.getDocValuesType() != null;
+ Bits bits = in.getDocsWithField(field);
+ assert bits != null;
+ assert bits.length() == maxDoc;
+ return new AssertingAtomicReader.AssertingBits(bits);
+ }
@Override
public void close() throws IOException {
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Fri Aug 30 15:06:42 2013
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesConsumer;
+import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingNormsConsumer;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat.AssertingDocValuesProducer;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.index.SegmentReadState;
@@ -38,7 +38,7 @@ public class AssertingNormsFormat extend
public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
DocValuesConsumer consumer = in.normsConsumer(state);
assert consumer != null;
- return new AssertingDocValuesConsumer(consumer, state.segmentInfo.getDocCount());
+ return new AssertingNormsConsumer(consumer, state.segmentInfo.getDocCount());
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java Fri Aug 30 15:06:42 2013
@@ -23,10 +23,12 @@ import org.apache.lucene.codecs.NormsFor
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat;
+import org.apache.lucene.codecs.diskdv.DiskNormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
/** Codec that tries to use as little ram as possible because he spent all his money on beer */
// TODO: better name :)
@@ -39,11 +41,11 @@ public class CheapBastardCodec extends F
private final StoredFieldsFormat storedFields = new Lucene40StoredFieldsFormat();
private final TermVectorsFormat termVectors = new Lucene40TermVectorsFormat();
// these go to disk for all docvalues/norms datastructures
- private final DocValuesFormat docValues = new CheapBastardDocValuesFormat();
- private final NormsFormat norms = new CheapBastardNormsFormat();
+ private final DocValuesFormat docValues = new DiskDocValuesFormat();
+ private final NormsFormat norms = new DiskNormsFormat();
public CheapBastardCodec() {
- super("CheapBastard", new Lucene42Codec());
+ super("CheapBastard", new Lucene45Codec());
}
public PostingsFormat postingsFormat() {
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java Fri Aug 30 15:06:42 2013
@@ -23,13 +23,13 @@ import org.apache.lucene.codecs.FilterCo
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec;
-import org.apache.lucene.codecs.lucene42.Lucene42Codec;
+import org.apache.lucene.codecs.lucene45.Lucene45Codec;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
/**
* A codec that uses {@link CompressingStoredFieldsFormat} for its stored
- * fields and delegates to {@link Lucene42Codec} for everything else.
+ * fields and delegates to {@link Lucene45Codec} for everything else.
*/
public abstract class CompressingCodec extends FilterCodec {
@@ -73,7 +73,7 @@ public abstract class CompressingCodec e
* Creates a compressing codec with a given segment suffix
*/
public CompressingCodec(String name, String segmentSuffix, CompressionMode compressionMode, int chunkSize) {
- super(name, new Lucene42Codec());
+ super(name, new Lucene45Codec());
this.storedFieldsFormat = new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize);
this.termVectorsFormat = new CompressingTermVectorsFormat(name, segmentSuffix, compressionMode, chunkSize);
}
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java Fri Aug 30 15:06:42 2013
@@ -1,8 +1,6 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.util.packed.PackedInts;
@@ -42,9 +40,4 @@ public class FastCompressingCodec extend
public NormsFormat normsFormat() {
return new Lucene42NormsFormat(PackedInts.FAST);
}
-
- @Override
- public DocValuesFormat docValuesFormat() {
- return new Lucene42DocValuesFormat(PackedInts.FAST);
- }
}
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java Fri Aug 30 15:06:42 2013
@@ -1,8 +1,6 @@
package org.apache.lucene.codecs.compressing;
-import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.util.packed.PackedInts;
@@ -42,9 +40,4 @@ public class FastDecompressionCompressin
public NormsFormat normsFormat() {
return new Lucene42NormsFormat(PackedInts.DEFAULT);
}
-
- @Override
- public DocValuesFormat docValuesFormat() {
- return new Lucene42DocValuesFormat(PackedInts.DEFAULT);
- }
}
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java Fri Aug 30 15:06:42 2013
@@ -24,7 +24,9 @@ import java.util.TreeSet;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.MissingOrdRemapper;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosReader.LegacyDocValuesType;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
@@ -54,7 +56,7 @@ class Lucene40DocValuesWriter extends Do
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for (Number n : values) {
- long v = n.longValue();
+ long v = n == null ? 0 : n.longValue();
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
}
@@ -92,7 +94,7 @@ class Lucene40DocValuesWriter extends Do
Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
output.writeInt(1); // size
for (Number n : values) {
- output.writeByte(n.byteValue());
+ output.writeByte(n == null ? 0 : n.byteValue());
}
}
@@ -103,7 +105,7 @@ class Lucene40DocValuesWriter extends Do
Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
output.writeInt(2); // size
for (Number n : values) {
- output.writeShort(n.shortValue());
+ output.writeShort(n == null ? 0 : n.shortValue());
}
}
@@ -114,7 +116,7 @@ class Lucene40DocValuesWriter extends Do
Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
output.writeInt(4); // size
for (Number n : values) {
- output.writeInt(n.intValue());
+ output.writeInt(n == null ? 0 : n.intValue());
}
}
@@ -131,7 +133,7 @@ class Lucene40DocValuesWriter extends Do
// writes longs
output.writeByte(Lucene40DocValuesFormat.VAR_INTS_FIXED_64);
for (Number n : values) {
- output.writeLong(n.longValue());
+ output.writeLong(n == null ? 0 : n.longValue());
}
} else {
// writes packed ints
@@ -143,7 +145,8 @@ class Lucene40DocValuesWriter extends Do
PackedInts.bitsRequired(delta),
PackedInts.DEFAULT);
for (Number n : values) {
- writer.add(n.longValue() - minValue);
+ long v = n == null ? 0 : n.longValue();
+ writer.add(v - minValue);
}
writer.finish();
}
@@ -156,6 +159,12 @@ class Lucene40DocValuesWriter extends Do
int minLength = Integer.MAX_VALUE;
int maxLength = Integer.MIN_VALUE;
for (BytesRef b : values) {
+ if (b == null) {
+ b = new BytesRef(); // 4.0 doesnt distinguish
+ }
+ if (b.length > Lucene40DocValuesFormat.MAX_BINARY_FIELD_LENGTH) {
+ throw new IllegalArgumentException("DocValuesField \"" + field.name + "\" is too large, must be <= " + Lucene40DocValuesFormat.MAX_BINARY_FIELD_LENGTH);
+ }
minLength = Math.min(minLength, b.length);
maxLength = Math.max(maxLength, b.length);
if (uniqueValues != null) {
@@ -243,7 +252,9 @@ class Lucene40DocValuesWriter extends Do
output.writeInt(length);
for (BytesRef v : values) {
- output.writeBytes(v.bytes, v.offset, v.length);
+ if (v != null) {
+ output.writeBytes(v.bytes, v.offset, v.length);
+ }
}
}
@@ -264,7 +275,9 @@ class Lucene40DocValuesWriter extends Do
final long startPos = data.getFilePointer();
for (BytesRef v : values) {
- data.writeBytes(v.bytes, v.offset, v.length);
+ if (v != null) {
+ data.writeBytes(v.bytes, v.offset, v.length);
+ }
}
/* addresses */
@@ -279,7 +292,9 @@ class Lucene40DocValuesWriter extends Do
long currentPosition = 0;
for (BytesRef v : values) {
w.add(currentPosition);
- currentPosition += v.length;
+ if (v != null) {
+ currentPosition += v.length;
+ }
}
// write sentinel
assert currentPosition == maxAddress;
@@ -301,7 +316,7 @@ class Lucene40DocValuesWriter extends Do
// deduplicate
TreeSet<BytesRef> dictionary = new TreeSet<BytesRef>();
for (BytesRef v : values) {
- dictionary.add(BytesRef.deepCopyOf(v));
+ dictionary.add(v == null ? new BytesRef() : BytesRef.deepCopyOf(v));
}
/* values */
@@ -318,6 +333,9 @@ class Lucene40DocValuesWriter extends Do
final PackedInts.Writer w = PackedInts.getWriter(index, maxDoc, PackedInts.bitsRequired(valueCount-1), PackedInts.DEFAULT);
for (BytesRef v : values) {
+ if (v == null) {
+ v = new BytesRef();
+ }
int ord = dictionary.headSet(v).size();
w.add(ord);
}
@@ -338,7 +356,7 @@ class Lucene40DocValuesWriter extends Do
// deduplicate
TreeSet<BytesRef> dictionary = new TreeSet<BytesRef>();
for (BytesRef v : values) {
- dictionary.add(BytesRef.deepCopyOf(v));
+ dictionary.add(v == null ? new BytesRef() : BytesRef.deepCopyOf(v));
}
/* values */
@@ -359,7 +377,7 @@ class Lucene40DocValuesWriter extends Do
final PackedInts.Writer w = PackedInts.getWriter(index, maxDoc, PackedInts.bitsRequired(currentAddress), PackedInts.DEFAULT);
for (BytesRef v : values) {
- w.add(valueToAddress.get(v));
+ w.add(valueToAddress.get(v == null ? new BytesRef() : v));
}
w.finish();
}
@@ -385,6 +403,15 @@ class Lucene40DocValuesWriter extends Do
maxLength = Math.max(maxLength, b.length);
}
+ // but dont use fixed if there are missing values (we are simulating how lucene40 wrote dv...)
+ boolean anyMissing = false;
+ for (Number n : docToOrd) {
+ if (n.longValue() == -1) {
+ anyMissing = true;
+ break;
+ }
+ }
+
boolean success = false;
IndexOutput data = null;
IndexOutput index = null;
@@ -394,12 +421,22 @@ class Lucene40DocValuesWriter extends Do
try {
data = dir.createOutput(dataName, state.context);
index = dir.createOutput(indexName, state.context);
- if (minLength == maxLength) {
+ if (minLength == maxLength && !anyMissing) {
// fixed byte[]
addFixedSortedBytesField(field, data, index, values, docToOrd, minLength);
} else {
// var byte[]
- addVarSortedBytesField(field, data, index, values, docToOrd);
+ // three cases for simulating the old writer:
+ // 1. no missing
+ // 2. missing (and empty string in use): remap ord=-1 -> ord=0
+ // 3. missing (and empty string not in use): remap all ords +1, insert empty string into values
+ if (!anyMissing) {
+ addVarSortedBytesField(field, data, index, values, docToOrd);
+ } else if (minLength == 0) {
+ addVarSortedBytesField(field, data, index, values, MissingOrdRemapper.mapMissingToOrd0(docToOrd));
+ } else {
+ addVarSortedBytesField(field, data, index, MissingOrdRemapper.insertEmptyValue(values), MissingOrdRemapper.mapAllOrds(docToOrd));
+ }
}
success = true;
} finally {
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1518989&r1=1518988&r2=1518989&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Fri Aug 30 15:06:42 2013
@@ -438,14 +438,14 @@ public class AssertingAtomicReader exten
this.in = in;
this.maxDoc = maxDoc;
this.valueCount = in.getValueCount();
- assert valueCount >= 1 && valueCount <= maxDoc;
+ assert valueCount >= 0 && valueCount <= maxDoc;
}
@Override
public int getOrd(int docID) {
assert docID >= 0 && docID < maxDoc;
int ord = in.getOrd(docID);
- assert ord >= 0 && ord < valueCount;
+ assert ord >= -1 && ord < valueCount;
return ord;
}
@@ -607,6 +607,54 @@ public class AssertingAtomicReader exten
return null;
}
}
+
+ /** Wraps a Bits but with additional asserts */
+ public static class AssertingBits implements Bits {
+ final Bits in;
+
+ public AssertingBits(Bits in) {
+ this.in = in;
+ }
+
+ @Override
+ public boolean get(int index) {
+ assert index >= 0 && index < length();
+ return in.get(index);
+ }
+
+ @Override
+ public int length() {
+ return in.length();
+ }
+ }
+
+ @Override
+ public Bits getLiveDocs() {
+ Bits liveDocs = super.getLiveDocs();
+ if (liveDocs != null) {
+ assert maxDoc() == liveDocs.length();
+ liveDocs = new AssertingBits(liveDocs);
+ } else {
+ assert maxDoc() == numDocs();
+ assert !hasDeletions();
+ }
+ return liveDocs;
+ }
+
+ @Override
+ public Bits getDocsWithField(String field) throws IOException {
+ Bits docsWithField = super.getDocsWithField(field);
+ FieldInfo fi = getFieldInfos().fieldInfo(field);
+ if (docsWithField != null) {
+ assert fi != null;
+ assert fi.hasDocValues();
+ assert maxDoc() == docsWithField.length();
+ docsWithField = new AssertingBits(docsWithField);
+ } else {
+ assert fi == null || fi.hasDocValues() == false;
+ }
+ return docsWithField;
+ }
// this is the same hack as FCInvisible
@Override