You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/07/16 19:00:18 UTC
svn commit: r1503797 [7/19] - in /lucene/dev/branches/lucene3069: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/suggest/
dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-tools/maven/ dev-tools/...
Modified: lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Tue Jul 16 17:00:05 2013
@@ -650,8 +650,11 @@ public class TestPackedInts extends Luce
wrt.set(99, (1 << 23) - 1);
assertEquals(1 << 10, wrt.get(valueCount - 1));
wrt.set(1, Long.MAX_VALUE);
+ wrt.set(2, -3);
+ assertEquals(64, wrt.getBitsPerValue());
assertEquals(1 << 10, wrt.get(valueCount - 1));
assertEquals(Long.MAX_VALUE, wrt.get(1));
+ assertEquals(-3L, wrt.get(2));
assertEquals(2, wrt.get(4));
assertEquals((1 << 23) - 1, wrt.get(99));
assertEquals(10, wrt.get(7));
@@ -686,7 +689,7 @@ public class TestPackedInts extends Luce
}
// test ramBytesUsed
- assertEquals(RamUsageEstimator.sizeOf(writer), writer.ramBytesUsed());
+ assertEquals(RamUsageEstimator.sizeOf(writer), writer.ramBytesUsed(), 8);
// test copy
PagedGrowableWriter copy = writer.resize(_TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
@@ -697,6 +700,64 @@ public class TestPackedInts extends Luce
assertEquals(0, copy.get(i));
}
}
+
+ // test grow
+ PagedGrowableWriter grow = writer.grow(_TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
+ for (long i = 0; i < grow.size(); ++i) {
+ if (i < writer.size()) {
+ assertEquals(writer.get(i), grow.get(i));
+ } else {
+ assertEquals(0, grow.get(i));
+ }
+ }
+ }
+
+ public void testPagedMutable() {
+ final int bitsPerValue = _TestUtil.nextInt(random(), 1, 64);
+ final long max = PackedInts.maxValue(bitsPerValue);
+ int pageSize = 1 << (_TestUtil.nextInt(random(), 6, 30));
+ // supports 0 values?
+ PagedMutable writer = new PagedMutable(0, pageSize, bitsPerValue, random().nextFloat() / 2);
+ assertEquals(0, writer.size());
+
+ // compare against AppendingLongBuffer
+ AppendingLongBuffer buf = new AppendingLongBuffer();
+ int size = random().nextInt(1000000);
+
+ for (int i = 0; i < size; ++i) {
+ buf.add(bitsPerValue == 64 ? random().nextLong() : _TestUtil.nextLong(random(), 0, max));
+ }
+ writer = new PagedMutable(size, pageSize, bitsPerValue, random().nextFloat());
+ assertEquals(size, writer.size());
+ for (int i = size - 1; i >= 0; --i) {
+ writer.set(i, buf.get(i));
+ }
+ for (int i = 0; i < size; ++i) {
+ assertEquals(buf.get(i), writer.get(i));
+ }
+
+ // test ramBytesUsed
+ assertEquals(RamUsageEstimator.sizeOf(writer) - RamUsageEstimator.sizeOf(writer.format), writer.ramBytesUsed());
+
+ // test copy
+ PagedMutable copy = writer.resize(_TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
+ for (long i = 0; i < copy.size(); ++i) {
+ if (i < writer.size()) {
+ assertEquals(writer.get(i), copy.get(i));
+ } else {
+ assertEquals(0, copy.get(i));
+ }
+ }
+
+ // test grow
+ PagedMutable grow = writer.grow(_TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
+ for (long i = 0; i < grow.size(); ++i) {
+ if (i < writer.size()) {
+ assertEquals(writer.get(i), grow.get(i));
+ } else {
+ assertEquals(0, grow.get(i));
+ }
+ }
}
// memory hole
@@ -897,6 +958,14 @@ public class TestPackedInts extends Luce
buf.add(arr[i]);
}
assertEquals(arr.length, buf.size());
+ if (random().nextBoolean()) {
+ buf.freeze();
+ if (random().nextBoolean()) {
+ // Make sure double freeze doesn't break anything
+ buf.freeze();
+ }
+ }
+ assertEquals(arr.length, buf.size());
final AbstractAppendingLongBuffer.Iterator it = buf.iterator();
for (int i = 0; i < arr.length; ++i) {
if (random().nextBoolean()) {
@@ -912,8 +981,7 @@ public class TestPackedInts extends Luce
final long expectedBytesUsed = RamUsageEstimator.sizeOf(buf);
final long computedBytesUsed = buf.ramBytesUsed();
- assertEquals("got " + computedBytesUsed + ", expected: " + expectedBytesUsed,
- expectedBytesUsed, computedBytesUsed);
+ assertEquals(expectedBytesUsed, computedBytesUsed);
}
}
}
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/range/RangeAccumulator.java Tue Jul 16 17:00:05 2013
@@ -95,6 +95,10 @@ public class RangeAccumulator extends Fa
int doc = 0;
while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) {
long v = ndv.get(doc);
+ // TODO: if all ranges are non-overlapping, we
+ // should instead do a bin-search up front
+ // (really, a specialized case of the interval
+ // tree)
// TODO: use interval tree instead of linear search:
for(int j=0;j<ranges.ranges.length;j++) {
if (ranges.ranges[j].accept(v)) {
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSideways.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSideways.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSideways.java Tue Jul 16 17:00:05 2013
@@ -208,9 +208,9 @@ public class DrillSideways {
requests.add(fr);
}
}
- if (requests.isEmpty()) {
- throw new IllegalArgumentException("could not find FacetRequest for drill-sideways dimension \"" + dim + "\"");
- }
+ // We already moved all drill-downs that didn't have a
+ // FacetRequest, in moveDrillDownOnlyClauses above:
+ assert !requests.isEmpty();
drillSidewaysCollectors[idx++] = FacetsCollector.create(getDrillSidewaysAccumulator(dim, new FacetSearchParams(fsp.indexingParams, requests)));
}
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysQuery.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysQuery.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysQuery.java Tue Jul 16 17:00:05 2013
@@ -119,6 +119,8 @@ class DrillSidewaysQuery extends Query {
Terms terms = reader.terms(field);
if (terms != null) {
termsEnum = terms.iterator(null);
+ } else {
+ termsEnum = null;
}
lastField = field;
}
@@ -129,7 +131,7 @@ class DrillSidewaysQuery extends Query {
}
for(int i=0;i<drillDownTerms[dim].length;i++) {
if (termsEnum.seekExact(drillDownTerms[dim][i].bytes(), false)) {
- DocsEnum docsEnum = termsEnum.docs(null, null);
+ DocsEnum docsEnum = termsEnum.docs(null, null, 0);
if (docsEnum != null) {
dims[dim].docsEnums[i] = docsEnum;
dims[dim].maxCost = Math.max(dims[dim].maxCost, docsEnum.cost());
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesAccumulator.java Tue Jul 16 17:00:05 2013
@@ -35,9 +35,11 @@ import org.apache.lucene.facet.search.Fa
import org.apache.lucene.facet.search.FacetsAggregator;
import org.apache.lucene.facet.search.FacetsCollector.MatchingDocs;
import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PriorityQueue;
@@ -57,7 +59,7 @@ public class SortedSetDocValuesAccumulat
final String field;
public SortedSetDocValuesAccumulator(FacetSearchParams fsp, SortedSetDocValuesReaderState state) throws IOException {
- super(fsp, null, null, new FacetArrays((int) state.getDocValues().getValueCount()));
+ super(fsp, null, null, new FacetArrays(state.getSize()));
this.state = state;
this.field = state.getField();
dv = state.getDocValues();
@@ -90,13 +92,23 @@ public class SortedSetDocValuesAccumulat
@Override
public void aggregate(MatchingDocs matchingDocs, CategoryListParams clp, FacetArrays facetArrays) throws IOException {
- SortedSetDocValues segValues = matchingDocs.context.reader().getSortedSetDocValues(field);
+ AtomicReader reader = matchingDocs.context.reader();
+
+ // LUCENE-5090: make sure the provided reader context "matches"
+ // the top-level reader passed to the
+ // SortedSetDocValuesReaderState, else cryptic
+ // AIOOBE can happen:
+ if (ReaderUtil.getTopLevelContext(matchingDocs.context).reader() != state.origReader) {
+ throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
+ }
+
+ SortedSetDocValues segValues = reader.getSortedSetDocValues(field);
if (segValues == null) {
return;
}
final int[] counts = facetArrays.getIntArray();
- final int maxDoc = matchingDocs.context.reader().maxDoc();
+ final int maxDoc = reader.maxDoc();
assert maxDoc == matchingDocs.bits.length();
if (dv instanceof MultiSortedSetDocValues) {
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesReaderState.java Tue Jul 16 17:00:05 2013
@@ -51,6 +51,7 @@ public final class SortedSetDocValuesRea
private final String field;
private final AtomicReader topReader;
private final int valueCount;
+ final IndexReader origReader;
final char separator;
final String separatorRegex;
@@ -91,6 +92,7 @@ public final class SortedSetDocValuesRea
this.field = fip.getCategoryListParams(null).field + FACET_FIELD_EXTENSION;
this.separator = fip.getFacetDelimChar();
this.separatorRegex = Pattern.quote(Character.toString(separator));
+ this.origReader = reader;
// We need this to create thread-safe MultiSortedSetDV
// per collector:
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java Tue Jul 16 17:00:05 2013
@@ -611,8 +611,9 @@ public class TestDrillSideways extends F
w.close();
final SortedSetDocValuesReaderState sortedSetDVState;
+ IndexSearcher s = newSearcher(r);
if (doUseDV) {
- sortedSetDVState = new SortedSetDocValuesReaderState(r);
+ sortedSetDVState = new SortedSetDocValuesReaderState(s.getIndexReader());
} else {
sortedSetDVState = null;
}
@@ -625,8 +626,6 @@ public class TestDrillSideways extends F
TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
tw.close();
- IndexSearcher s = newSearcher(r);
-
int numIters = atLeast(10);
for(int iter=0;iter<numIters;iter++) {
Modified: lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java (original)
+++ lucene/dev/branches/lucene3069/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java Tue Jul 16 17:00:05 2013
@@ -33,6 +33,7 @@ import org.apache.lucene.facet.search.Fa
import org.apache.lucene.facet.search.FacetResult;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -145,4 +146,51 @@ public class TestSortedSetDocValuesFacet
searcher.getIndexReader().close();
dir.close();
}
+
+ // LUCENE-5090
+ public void testStaleState() throws Exception {
+ assumeTrue("Test requires SortedSetDV support", defaultCodecSupportsSortedSet());
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+
+ SortedSetDocValuesFacetFields dvFields = new SortedSetDocValuesFacetFields();
+
+ Document doc = new Document();
+ dvFields.addFields(doc, Collections.singletonList(new CategoryPath("a", "foo")));
+ writer.addDocument(doc);
+
+ IndexReader r = writer.getReader();
+ SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(r);
+
+ doc = new Document();
+ dvFields.addFields(doc, Collections.singletonList(new CategoryPath("a", "bar")));
+ writer.addDocument(doc);
+
+ doc = new Document();
+ dvFields.addFields(doc, Collections.singletonList(new CategoryPath("a", "baz")));
+ writer.addDocument(doc);
+
+ IndexSearcher searcher = newSearcher(writer.getReader());
+
+ List<FacetRequest> requests = new ArrayList<FacetRequest>();
+ requests.add(new CountFacetRequest(new CategoryPath("a"), 10));
+
+ FacetSearchParams fsp = new FacetSearchParams(requests);
+
+ FacetsCollector c = FacetsCollector.create(new SortedSetDocValuesAccumulator(fsp, state));
+
+ searcher.search(new MatchAllDocsQuery(), c);
+
+ try {
+ c.getFacetResults();
+ fail("did not hit expected exception");
+ } catch (IllegalStateException ise) {
+ // expected
+ }
+
+ r.close();
+ writer.close();
+ searcher.getIndexReader().close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java Tue Jul 16 17:00:05 2013
@@ -78,7 +78,7 @@ public class AllGroupHeadsCollectorTest
Document doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "random text", Field.Store.NO));
- doc.add(newStringField("id_1", "1", Field.Store.NO));
+ doc.add(new IntField("id_1", 1, Field.Store.NO));
doc.add(newStringField("id_2", "1", Field.Store.NO));
w.addDocument(doc);
@@ -86,7 +86,7 @@ public class AllGroupHeadsCollectorTest
doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "some more random text blob", Field.Store.NO));
- doc.add(newStringField("id_1", "2", Field.Store.NO));
+ doc.add(new IntField("id_1", 2, Field.Store.NO));
doc.add(newStringField("id_2", "2", Field.Store.NO));
w.addDocument(doc);
@@ -94,7 +94,7 @@ public class AllGroupHeadsCollectorTest
doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "some more random textual data", Field.Store.NO));
- doc.add(newStringField("id_1", "3", Field.Store.NO));
+ doc.add(new IntField("id_1", 3, Field.Store.NO));
doc.add(newStringField("id_2", "3", Field.Store.NO));
w.addDocument(doc);
w.commit(); // To ensure a second segment
@@ -103,7 +103,7 @@ public class AllGroupHeadsCollectorTest
doc = new Document();
addGroupField(doc, groupField, "author2", valueType);
doc.add(newTextField("content", "some random text", Field.Store.NO));
- doc.add(newStringField("id_1", "4", Field.Store.NO));
+ doc.add(new IntField("id_1", 4, Field.Store.NO));
doc.add(newStringField("id_2", "4", Field.Store.NO));
w.addDocument(doc);
@@ -111,7 +111,7 @@ public class AllGroupHeadsCollectorTest
doc = new Document();
addGroupField(doc, groupField, "author3", valueType);
doc.add(newTextField("content", "some more random text", Field.Store.NO));
- doc.add(newStringField("id_1", "5", Field.Store.NO));
+ doc.add(new IntField("id_1", 5, Field.Store.NO));
doc.add(newStringField("id_2", "5", Field.Store.NO));
w.addDocument(doc);
@@ -119,21 +119,21 @@ public class AllGroupHeadsCollectorTest
doc = new Document();
addGroupField(doc, groupField, "author3", valueType);
doc.add(newTextField("content", "random blob", Field.Store.NO));
- doc.add(newStringField("id_1", "6", Field.Store.NO));
+ doc.add(new IntField("id_1", 6, Field.Store.NO));
doc.add(newStringField("id_2", "6", Field.Store.NO));
w.addDocument(doc);
// 6 -- no author field
doc = new Document();
doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
- doc.add(newStringField("id_1", "6", Field.Store.NO));
+ doc.add(new IntField("id_1", 6, Field.Store.NO));
doc.add(newStringField("id_2", "6", Field.Store.NO));
w.addDocument(doc);
// 7 -- no author field
doc = new Document();
doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
- doc.add(newStringField("id_1", "7", Field.Store.NO));
+ doc.add(new IntField("id_1", 7, Field.Store.NO));
doc.add(newStringField("id_2", "7", Field.Store.NO));
w.addDocument(doc);
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Tue Jul 16 17:00:05 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.search.highlig
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Iterator;
@@ -25,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.PriorityQueue;
/**
@@ -78,7 +76,7 @@ public class Highlighter
public final String getBestFragment(Analyzer analyzer, String fieldName,String text)
throws IOException, InvalidTokenOffsetsException
{
- TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(fieldName, text);
return getBestFragment(tokenStream, text);
}
@@ -130,7 +128,7 @@ public class Highlighter
int maxNumFragments)
throws IOException, InvalidTokenOffsetsException
{
- TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(fieldName, text);
return getBestFragments(tokenStream, text, maxNumFragments);
}
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Tue Jul 16 17:00:05 2013
@@ -36,7 +36,6 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Comparator;
@@ -314,7 +313,7 @@ public class TokenSources {
public static TokenStream getTokenStream(String field, String contents,
Analyzer analyzer) {
try {
- return analyzer.tokenStream(field, new StringReader(contents));
+ return analyzer.tokenStream(field, contents);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java Tue Jul 16 17:00:05 2013
@@ -369,7 +369,11 @@ public class PostingsHighlighter {
* identical to what was indexed. */
protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[] docids, int maxLength) throws IOException {
String contents[][] = new String[fields.length][docids.length];
- LimitedStoredFieldVisitor visitor = new LimitedStoredFieldVisitor(fields, maxLength);
+ char valueSeparators[] = new char[fields.length];
+ for (int i = 0; i < fields.length; i++) {
+ valueSeparators[i] = getMultiValuedSeparator(fields[i]);
+ }
+ LimitedStoredFieldVisitor visitor = new LimitedStoredFieldVisitor(fields, valueSeparators, maxLength);
for (int i = 0; i < docids.length; i++) {
searcher.doc(docids[i], visitor);
for (int j = 0; j < fields.length; j++) {
@@ -379,6 +383,16 @@ public class PostingsHighlighter {
}
return contents;
}
+
+ /**
+ * Returns the logical separator between values for multi-valued fields.
+ * The default value is a space character, which means passages can span across values,
+ * but a subclass can override, for example with {@code U+2029 PARAGRAPH SEPARATOR (PS)}
+ * if each value holds a discrete passage for highlighting.
+ */
+ protected char getMultiValuedSeparator(String field) {
+ return ' ';
+ }
private Map<Integer,String> highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List<AtomicReaderContext> leaves, int maxPassages) throws IOException {
Map<Integer,String> highlights = new HashMap<Integer,String>();
@@ -652,12 +666,15 @@ public class PostingsHighlighter {
private static class LimitedStoredFieldVisitor extends StoredFieldVisitor {
private final String fields[];
+ private final char valueSeparators[];
private final int maxLength;
private final StringBuilder builders[];
private int currentField = -1;
- public LimitedStoredFieldVisitor(String fields[], int maxLength) {
+ public LimitedStoredFieldVisitor(String fields[], char valueSeparators[], int maxLength) {
+ assert fields.length == valueSeparators.length;
this.fields = fields;
+ this.valueSeparators = valueSeparators;
this.maxLength = maxLength;
builders = new StringBuilder[fields.length];
for (int i = 0; i < builders.length; i++) {
@@ -670,7 +687,7 @@ public class PostingsHighlighter {
assert currentField >= 0;
StringBuilder builder = builders[currentField];
if (builder.length() > 0 && builder.length() < maxLength) {
- builder.append(' '); // for the offset gap, TODO: make this configurable
+ builder.append(valueSeparators[currentField]);
}
if (builder.length() + value.length() > maxLength) {
builder.append(value, 0, maxLength - builder.length());
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Tue Jul 16 17:00:05 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.search.highlig
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -248,7 +247,7 @@ public class HighlighterTest extends Bas
private String highlightField(Query query, String fieldName, String text)
throws IOException, InvalidTokenOffsetsException {
TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)
- .tokenStream(fieldName, new StringReader(text));
+ .tokenStream(fieldName, text);
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
@@ -269,8 +268,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
- new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
@@ -348,7 +346,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -377,7 +375,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -406,7 +404,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -431,7 +429,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -455,7 +453,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -479,7 +477,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -505,7 +503,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).getField(NUMERIC_FIELD_NAME).numericValue().toString();
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -533,7 +531,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -555,7 +553,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -585,7 +583,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
@@ -608,7 +606,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
@@ -639,7 +637,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -710,7 +708,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -907,7 +905,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -931,7 +929,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, null);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -955,7 +953,7 @@ public class HighlighterTest extends Bas
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
QueryScorer scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -1126,7 +1124,7 @@ public class HighlighterTest extends Bas
numHighlights = 0;
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
@@ -1187,7 +1185,7 @@ public class HighlighterTest extends Bas
Highlighter highlighter = getHighlighter(wTerms, HighlighterTest.this);// new
// Highlighter(new
// QueryTermScorer(wTerms));
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);
highlighter.setTextFragmenter(new SimpleFragmenter(2));
String result = highlighter.getBestFragment(tokenStream, texts[0]).trim();
@@ -1196,7 +1194,7 @@ public class HighlighterTest extends Bas
// readjust weights
wTerms[1].setWeight(50f);
- tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
+ tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);
highlighter = getHighlighter(wTerms, HighlighterTest.this);
highlighter.setTextFragmenter(new SimpleFragmenter(2));
@@ -1232,7 +1230,7 @@ public class HighlighterTest extends Bas
Highlighter highlighter = getHighlighter(query, null, HighlighterTest.this);
// Get 3 best fragments and separate with a "..."
- TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(s));
+ TokenStream tokenStream = analyzer.tokenStream(null, s);
String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
String expectedResult = "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";
@@ -1257,7 +1255,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
String result = highlighter.getBestFragment(tokenStream, text);
@@ -1280,7 +1278,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);// new Highlighter(this, new
@@ -1288,7 +1286,7 @@ public class HighlighterTest extends Bas
highlighter.setTextFragmenter(new SimpleFragmenter(20));
String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);
- tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ tokenStream = analyzer.tokenStream(FIELD_NAME, text);
TextFragment fragmentResults[] = highlighter.getBestTextFragments(tokenStream, text,
true, 10);
@@ -1318,7 +1316,7 @@ public class HighlighterTest extends Bas
public void run() throws Exception {
numHighlights = 0;
doSearching(new TermQuery(new Term(FIELD_NAME, "meat")));
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, texts[0]);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);// new Highlighter(this, new
// QueryTermScorer(query));
@@ -1432,7 +1430,7 @@ public class HighlighterTest extends Bas
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME, HighlighterTest.this, false);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
@@ -1461,7 +1459,7 @@ public class HighlighterTest extends Bas
doSearching(new TermQuery(new Term(FIELD_NAME, "aninvalidquerywhichshouldyieldnoresults")));
for (String text : texts) {
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
Highlighter highlighter = getHighlighter(query, FIELD_NAME,
HighlighterTest.this);
String result = highlighter.getBestFragment(tokenStream, text);
@@ -1503,7 +1501,7 @@ public class HighlighterTest extends Bas
}
});
highlighter.setTextFragmenter(new SimpleFragmenter(2000));
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(rawDocContent));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, rawDocContent);
String encodedSnippet = highlighter.getBestFragments(tokenStream, rawDocContent, 1, "");
// An ugly bit of XML creation:
@@ -1828,7 +1826,7 @@ public class HighlighterTest extends Bas
*
* for (int i = 0; i < hits.totalHits; i++) { String text =
* searcher.doc2(hits.scoreDocs[i].doc).get(FIELD_NAME); TokenStream
- * tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,new StringReader(text));
+ * tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,text);
* String highlightedText = highlighter.getBestFragment(tokenStream,text);
* System.out.println(highlightedText); } }
*/
@@ -1855,7 +1853,7 @@ public class HighlighterTest extends Bas
final int expectedHighlights) throws Exception {
for (int i = 0; i < hits.totalHits; i++) {
String text = searcher.doc(hits.scoreDocs[i].doc).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
Highlighter highlighter = new Highlighter(this, scorer);
@@ -2104,7 +2102,7 @@ final class SynonymTokenizer extends Tok
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
Scorer scorer = null;
- TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, text);
if (mode == QUERY) {
scorer = new QueryScorer(query);
} else if (mode == QUERY_TERM) {
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java Tue Jul 16 17:00:05 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.search.highlig
* limitations under the License.
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -89,8 +88,7 @@ public class HighlightCustomQueryTest ex
private String highlightField(Query query, String fieldName,
String text) throws IOException, InvalidTokenOffsetsException {
TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE,
- true, MockTokenFilter.ENGLISH_STOPSET).tokenStream(fieldName,
- new StringReader(text));
+ true, MockTokenFilter.ENGLISH_STOPSET).tokenStream(fieldName, text);
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
MyQueryScorer scorer = new MyQueryScorer(query, fieldName, FIELD_NAME);
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java Tue Jul 16 17:00:05 2013
@@ -922,4 +922,48 @@ public class TestPostingsHighlighter ext
ir.close();
dir.close();
}
+
+ /** customizing the gap separator to force a sentence break */
+ public void testGapSeparator() throws Exception {
+ Directory dir = newDirectory();
+ // use simpleanalyzer for more natural tokenization (else "test." is a token)
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+ FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
+ offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ Document doc = new Document();
+
+ Field body1 = new Field("body", "", offsetsType);
+ body1.setStringValue("This is a multivalued field");
+ doc.add(body1);
+
+ Field body2 = new Field("body", "", offsetsType);
+ body2.setStringValue("This is something different");
+ doc.add(body2);
+
+ iw.addDocument(doc);
+
+ IndexReader ir = iw.getReader();
+ iw.close();
+
+ IndexSearcher searcher = newSearcher(ir);
+ PostingsHighlighter highlighter = new PostingsHighlighter() {
+ @Override
+ protected char getMultiValuedSeparator(String field) {
+ assert field.equals("body");
+ return '\u2029';
+ }
+ };
+ Query query = new TermQuery(new Term("body", "field"));
+ TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER);
+ assertEquals(1, topDocs.totalHits);
+ String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
+ assertEquals(1, snippets.length);
+ assertEquals("This is a multivalued <b>field</b>\u2029", snippets[0]);
+
+ ir.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/lucene3069/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Tue Jul 16 17:00:05 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.search.vectorh
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -171,7 +170,7 @@ public abstract class AbstractTestCase e
protected List<BytesRef> analyze(String text, String field, Analyzer analyzer) throws IOException {
List<BytesRef> bytesRefs = new ArrayList<BytesRef>();
- TokenStream tokenStream = analyzer.tokenStream(field, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(field, text);
TermToBytesRefAttribute termAttribute = tokenStream.getAttribute(TermToBytesRefAttribute.class);
BytesRef bytesRef = termAttribute.getBytesRef();
Modified: lucene/dev/branches/lucene3069/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (original)
+++ lucene/dev/branches/lucene3069/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java Tue Jul 16 17:00:05 2013
@@ -216,7 +216,7 @@ class TermsIncludingScoreQuery extends Q
return scores[ords[scoreUpto]];
}
- public Explanation explain() throws IOException {
+ Explanation explain() throws IOException {
return new ComplexExplanation(true, score(), "Score based on join value " + termsEnum.term().utf8ToString());
}
@@ -226,16 +226,16 @@ class TermsIncludingScoreQuery extends Q
}
int nextDocOutOfOrder() throws IOException {
- if (docsEnum != null) {
- int docId = docsEnum.nextDoc();
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- docsEnum = null;
- } else {
- return doc = docId;
+ while (true) {
+ if (docsEnum != null) {
+ int docId = docsEnumNextDoc();
+ if (docId == DocIdSetIterator.NO_MORE_DOCS) {
+ docsEnum = null;
+ } else {
+ return doc = docId;
+ }
}
- }
- do {
if (upto == terms.size()) {
return doc = DocIdSetIterator.NO_MORE_DOCS;
}
@@ -244,9 +244,11 @@ class TermsIncludingScoreQuery extends Q
if (termsEnum.seekExact(terms.get(ords[upto++], spare), true)) {
docsEnum = reuse = termsEnum.docs(acceptDocs, reuse, DocsEnum.FLAG_NONE);
}
- } while (docsEnum == null);
+ }
+ }
- return doc = docsEnum.nextDoc();
+ protected int docsEnumNextDoc() throws IOException {
+ return docsEnum.nextDoc();
}
@Override
@@ -301,47 +303,14 @@ class TermsIncludingScoreQuery extends Q
}
@Override
- int nextDocOutOfOrder() throws IOException {
- if (docsEnum != null) {
- int docId;
- do {
- docId = docsEnum.nextDoc();
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- } while (alreadyEmittedDocs.get(docId));
+ protected int docsEnumNextDoc() throws IOException {
+ while (true) {
+ int docId = docsEnum.nextDoc();
if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- docsEnum = null;
- } else {
- alreadyEmittedDocs.set(docId);
return docId;
}
- }
-
- for (;;) {
- do {
- if (upto == terms.size()) {
- return DocIdSetIterator.NO_MORE_DOCS;
- }
-
- scoreUpto = upto;
- if (termsEnum.seekExact(terms.get(ords[upto++], spare), true)) {
- docsEnum = reuse = termsEnum.docs(acceptDocs, reuse, DocsEnum.FLAG_NONE);
- }
- } while (docsEnum == null);
-
- int docId;
- do {
- docId = docsEnum.nextDoc();
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- } while (alreadyEmittedDocs.get(docId));
- if (docId == DocIdSetIterator.NO_MORE_DOCS) {
- docsEnum = null;
- } else {
- alreadyEmittedDocs.set(docId);
- return docId;
+ if (!alreadyEmittedDocs.getAndSet(docId)) {
+ return docId;//if it wasn't previously set, return it
}
}
}
Modified: lucene/dev/branches/lucene3069/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (original)
+++ lucene/dev/branches/lucene3069/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java Tue Jul 16 17:00:05 2013
@@ -459,7 +459,7 @@ public class TestBlockJoin extends Lucen
for(int parentDocID=0;parentDocID<numParentDocs;parentDocID++) {
Document parentDoc = new Document();
Document parentJoinDoc = new Document();
- Field id = newStringField("parentID", ""+parentDocID, Field.Store.YES);
+ Field id = new IntField("parentID", parentDocID, Field.Store.YES);
parentDoc.add(id);
parentJoinDoc.add(id);
parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
@@ -472,8 +472,8 @@ public class TestBlockJoin extends Lucen
}
if (doDeletes) {
- parentDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
- parentJoinDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
+ parentDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
+ parentJoinDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
}
final List<Document> joinDocs = new ArrayList<>();
@@ -497,7 +497,7 @@ public class TestBlockJoin extends Lucen
Document joinChildDoc = new Document();
joinDocs.add(joinChildDoc);
- Field childID = newStringField("childID", ""+childDocID, Field.Store.YES);
+ Field childID = new IntField("childID", childDocID, Field.Store.YES);
childDoc.add(childID);
joinChildDoc.add(childID);
@@ -522,7 +522,7 @@ public class TestBlockJoin extends Lucen
}
if (doDeletes) {
- joinChildDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
+ joinChildDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
}
w.addDocument(childDoc);
@@ -541,8 +541,10 @@ public class TestBlockJoin extends Lucen
if (VERBOSE) {
System.out.println("DELETE parentID=" + deleteID);
}
- w.deleteDocuments(new Term("blockID", ""+deleteID));
- joinW.deleteDocuments(new Term("blockID", ""+deleteID));
+ BytesRef term = new BytesRef();
+ NumericUtils.intToPrefixCodedBytes(deleteID, 0, term);
+ w.deleteDocuments(new Term("blockID", term));
+ joinW.deleteDocuments(new Term("blockID", term));
}
final IndexReader r = w.getReader();
Modified: lucene/dev/branches/lucene3069/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt (original)
+++ lucene/dev/branches/lucene3069/lucene/licenses/morfologik-fsa-LICENSE-BSD.txt Tue Jul 16 17:00:05 2013
@@ -1,6 +1,6 @@
Copyright (c) 2006 Dawid Weiss
-Copyright (c) 2007-2012 Dawid Weiss, Marcin MiÅkowski
+Copyright (c) 2007-2013 Dawid Weiss, Marcin MiÅkowski
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
@@ -26,4 +26,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIA
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
Modified: lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-LICENSE-BSD.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-LICENSE-BSD.txt?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-LICENSE-BSD.txt (original)
+++ lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-LICENSE-BSD.txt Tue Jul 16 17:00:05 2013
@@ -1,62 +1,26 @@
BSD-licensed dictionary of Polish (Morfologik)
-Copyright (c) 2012, Marcin MiÅkowski
+Morfologik Polish dictionary.
+Version: 2.0 PoliMorf
+Copyright (c) 2013, Marcin MiÅkowski
All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the
- distribution.
-
-THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS âAS ISâ AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
-BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---
-
-BSD-licensed dictionary of Polish (SGJP)
-http://sgjp.pl/morfeusz/
-
-Copyright © 2011 Zygmunt Saloni, WÅodzimierz GruszczyÅski,
- Marcin WoliÅski, Robert WoÅosz
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the
- distribution.
-
-THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS âAS ISâ AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
-BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Modified: lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-NOTICE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-NOTICE.txt?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-NOTICE.txt (original)
+++ lucene/dev/branches/lucene3069/lucene/licenses/morfologik-polish-NOTICE.txt Tue Jul 16 17:00:05 2013
@@ -1,6 +1,3 @@
-This product includes data from BSD-licensed dictionary of Polish (Morfologik)
-(http://morfologik.blogspot.com/)
-
-This product includes data from BSD-licensed dictionary of Polish (SGJP)
-(http://sgjp.pl/morfeusz/)
+This product includes data from BSD-licensed dictionary of Polish (Morfologik, PoliMorf)
+(http://morfologik.blogspot.com/)
\ No newline at end of file
Modified: lucene/dev/branches/lucene3069/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt (original)
+++ lucene/dev/branches/lucene3069/lucene/licenses/morfologik-stemming-LICENSE-BSD.txt Tue Jul 16 17:00:05 2013
@@ -1,6 +1,6 @@
Copyright (c) 2006 Dawid Weiss
-Copyright (c) 2007-2012 Dawid Weiss, Marcin MiÅkowski
+Copyright (c) 2007-2013 Dawid Weiss, Marcin MiÅkowski
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
@@ -26,4 +26,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIA
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
Modified: lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene3069/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Jul 16 17:00:05 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.index.memory;
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
@@ -291,7 +290,7 @@ public class MemoryIndex {
TokenStream stream;
try {
- stream = analyzer.tokenStream(fieldName, new StringReader(text));
+ stream = analyzer.tokenStream(fieldName, text);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
Modified: lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Tue Jul 16 17:00:05 2013
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
-import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
@@ -378,7 +377,7 @@ public class MemoryIndexTest extends Bas
SpanQuery wrappedquery = new SpanMultiTermQueryWrapper<RegexpQuery>(regex);
MemoryIndex mindex = new MemoryIndex(random().nextBoolean(), random().nextInt(50) * 1024 * 1024);
- mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", new StringReader("hello there")));
+ mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there"));
// This throws an NPE
assertEquals(0, mindex.search(wrappedquery), 0.00001f);
@@ -390,7 +389,7 @@ public class MemoryIndexTest extends Bas
SpanQuery wrappedquery = new SpanOrQuery(new SpanMultiTermQueryWrapper<RegexpQuery>(regex));
MemoryIndex mindex = new MemoryIndex(random().nextBoolean(), random().nextInt(50) * 1024 * 1024);
- mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", new StringReader("hello there")));
+ mindex.addField("field", new MockAnalyzer(random()).tokenStream("field", "hello there"));
// This passes though
assertEquals(0, mindex.search(wrappedquery), 0.00001f);
Modified: lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java Tue Jul 16 17:00:05 2013
@@ -191,6 +191,7 @@ public abstract class Sorter {
for (int i = 0; i < maxDoc; ++i) {
newToOld.add(docs[i]);
}
+ newToOld.freeze();
for (int i = 0; i < maxDoc; ++i) {
docs[(int) newToOld.get(i)] = i;
@@ -200,6 +201,7 @@ public abstract class Sorter {
for (int i = 0; i < maxDoc; ++i) {
oldToNew.add(docs[i]);
}
+ oldToNew.freeze();
return new Sorter.DocMap() {
Modified: lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java (original)
+++ lucene/dev/branches/lucene3069/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java Tue Jul 16 17:00:05 2013
@@ -107,6 +107,7 @@ public final class SortingMergePolicy ex
}
}
}
+ deletes.freeze();
return deletes;
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java Tue Jul 16 17:00:05 2013
@@ -29,7 +29,7 @@ import java.io.IOException;
import java.util.Map;
/**
- * Function that returns {@link TFIDFSimilarity#decodeNormValue(byte)}
+ * Function that returns {@link TFIDFSimilarity#decodeNormValue(long)}
* for every document.
* <p>
* Note that the configured Similarity for the field must be
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java Tue Jul 16 17:00:05 2013
@@ -53,26 +53,6 @@ public class TestCustomScoreQuery extend
}
/**
- * Test that CustomScoreQuery of Type.BYTE returns the expected scores.
- */
- @Test
- public void testCustomScoreByte() throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestCustomScore(BYTE_VALUESOURCE, 1.0);
- doTestCustomScore(BYTE_VALUESOURCE, 2.0);
- }
-
- /**
- * Test that CustomScoreQuery of Type.SHORT returns the expected scores.
- */
- @Test
- public void testCustomScoreShort() throws Exception {
- // INT field values are small enough to be parsed as short
- doTestCustomScore(SHORT_VALUESOURCE, 1.0);
- doTestCustomScore(SHORT_VALUESOURCE, 3.0);
- }
-
- /**
* Test that CustomScoreQuery of Type.INT returns the expected scores.
*/
@Test
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java Tue Jul 16 17:00:05 2013
@@ -1,18 +1,25 @@
package org.apache.lucene.queries.function;
+import java.io.IOException;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
@@ -53,10 +60,21 @@ public abstract class FunctionTestSetup
protected static final String INT_FIELD = "iii";
protected static final String FLOAT_FIELD = "fff";
- protected ValueSource BYTE_VALUESOURCE = new ByteFieldSource(INT_FIELD);
- protected ValueSource SHORT_VALUESOURCE = new ShortFieldSource(INT_FIELD);
+ private static final FieldCache.FloatParser CUSTOM_FLOAT_PARSER = new FieldCache.FloatParser() {
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return FieldCache.NUMERIC_UTILS_INT_PARSER.termsEnum(terms);
+ }
+
+ @Override
+ public float parseFloat(BytesRef term) {
+ return (float) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term);
+ }
+ };
+
protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
- protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD);
+ protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD, CUSTOM_FLOAT_PARSER);
protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
private static final String DOC_TEXT_LINES[] = {
@@ -140,10 +158,10 @@ public abstract class FunctionTestSetup
f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
d.add(f);
- f = newField(INT_FIELD, "" + scoreAndID, customType); // for function scoring
+ f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
- f = newField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring
+ f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
iw.addDocument(d);
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java Tue Jul 16 17:00:05 2013
@@ -19,12 +19,6 @@ package org.apache.lucene.queries.functi
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.queries.function.FunctionQuery;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
-import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
-import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
@@ -50,20 +44,6 @@ public class TestFieldScoreQuery extends
createIndex(true);
}
- /** Test that FieldScoreQuery of Type.BYTE returns docs in expected order. */
- @Test
- public void testRankByte () throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestRank(BYTE_VALUESOURCE);
- }
-
- /** Test that FieldScoreQuery of Type.SHORT returns docs in expected order. */
- @Test
- public void testRankShort () throws Exception {
- // INT field values are small enough to be parsed as short
- doTestRank(SHORT_VALUESOURCE);
- }
-
/** Test that FieldScoreQuery of Type.INT returns docs in expected order. */
@Test
public void testRankInt () throws Exception {
@@ -99,20 +79,6 @@ public class TestFieldScoreQuery extends
r.close();
}
- /** Test that FieldScoreQuery of Type.BYTE returns the expected scores. */
- @Test
- public void testExactScoreByte () throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestExactScore(BYTE_VALUESOURCE);
- }
-
- /** Test that FieldScoreQuery of Type.SHORT returns the expected scores. */
- @Test
- public void testExactScoreShort () throws Exception {
- // INT field values are small enough to be parsed as short
- doTestExactScore(SHORT_VALUESOURCE);
- }
-
/** Test that FieldScoreQuery of Type.INT returns the expected scores. */
@Test
public void testExactScoreInt () throws Exception {
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java Tue Jul 16 17:00:05 2013
@@ -18,9 +18,10 @@ package org.apache.lucene.queries.functi
*/
import java.io.IOException;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@@ -46,13 +47,13 @@ public class TestFunctionQuerySort exten
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
- Field field = new StringField("value", "", Field.Store.YES);
+ Field field = new IntField("value", 0, Field.Store.YES);
doc.add(field);
// Save docs unsorted (decreasing value n, n-1, ...)
final int NUM_VALS = 5;
for (int val = NUM_VALS; val > 0; val--) {
- field.setStringValue(Integer.toString(val));
+ field.setIntValue(val);
writer.addDocument(doc);
}
Modified: lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java Tue Jul 16 17:00:05 2013
@@ -22,13 +22,16 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.queries.function.valuesource.ConstValueSource;
import org.apache.lucene.queries.function.valuesource.DivFloatFunction;
@@ -54,7 +57,6 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.RangeMapFloatFunction;
import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction;
import org.apache.lucene.queries.function.valuesource.ScaleFloatFunction;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.queries.function.valuesource.SumFloatFunction;
import org.apache.lucene.queries.function.valuesource.SumTotalTermFreqValueSource;
import org.apache.lucene.queries.function.valuesource.TFValueSource;
@@ -85,9 +87,9 @@ public class TestValueSources extends Lu
static IndexSearcher searcher;
static final List<String[]> documents = Arrays.asList(new String[][] {
- /* id, byte, double, float, int, long, short, string, text */
- new String[] { "0", "5", "3.63", "5.2", "35", "4343", "945", "test", "this is a test test test" },
- new String[] { "1", "12", "5.65", "9.3", "54", "1954", "123", "bar", "second test" },
+ /* id, double, float, int, long, string, text */
+ new String[] { "0", "3.63", "5.2", "35", "4343", "test", "this is a test test test" },
+ new String[] { "1", "5.65", "9.3", "54", "1954", "bar", "second test" },
});
@BeforeClass
@@ -99,18 +101,14 @@ public class TestValueSources extends Lu
Document document = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
document.add(idField);
- Field byteField = new StringField("byte", "", Field.Store.NO);
- document.add(byteField);
- Field doubleField = new StringField("double", "", Field.Store.NO);
+ Field doubleField = new DoubleField("double", 0d, Field.Store.NO);
document.add(doubleField);
- Field floatField = new StringField("float", "", Field.Store.NO);
+ Field floatField = new FloatField("float", 0f, Field.Store.NO);
document.add(floatField);
- Field intField = new StringField("int", "", Field.Store.NO);
+ Field intField = new IntField("int", 0, Field.Store.NO);
document.add(intField);
- Field longField = new StringField("long", "", Field.Store.NO);
+ Field longField = new LongField("long", 0L, Field.Store.NO);
document.add(longField);
- Field shortField = new StringField("short", "", Field.Store.NO);
- document.add(shortField);
Field stringField = new StringField("string", "", Field.Store.NO);
document.add(stringField);
Field textField = new TextField("text", "", Field.Store.NO);
@@ -118,14 +116,12 @@ public class TestValueSources extends Lu
for (String [] doc : documents) {
idField.setStringValue(doc[0]);
- byteField.setStringValue(doc[1]);
- doubleField.setStringValue(doc[2]);
- floatField.setStringValue(doc[3]);
- intField.setStringValue(doc[4]);
- longField.setStringValue(doc[5]);
- shortField.setStringValue(doc[6]);
- stringField.setStringValue(doc[7]);
- textField.setStringValue(doc[8]);
+ doubleField.setDoubleValue(Double.valueOf(doc[1]));
+ floatField.setFloatValue(Float.valueOf(doc[2]));
+ intField.setIntValue(Integer.valueOf(doc[3]));
+ longField.setLongValue(Long.valueOf(doc[4]));
+ stringField.setStringValue(doc[5]);
+ textField.setStringValue(doc[6]);
iw.addDocument(document);
}
@@ -143,11 +139,6 @@ public class TestValueSources extends Lu
dir = null;
}
- public void testByte() throws Exception {
- assertHits(new FunctionQuery(new ByteFieldSource("byte")),
- new float[] { 5f, 12f });
- }
-
public void testConst() throws Exception {
assertHits(new FunctionQuery(new ConstValueSource(0.3f)),
new float[] { 0.3f, 0.3f });
@@ -298,11 +289,6 @@ public class TestValueSources extends Lu
new float[] { 0.0f, 1.0f });
}
- public void testShort() throws Exception {
- assertHits(new FunctionQuery(new ShortFieldSource("short")),
- new float[] { 945f, 123f });
- }
-
public void testSumFloat() throws Exception {
assertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] {
new ConstValueSource(1f), new ConstValueSource(2f)})),
Modified: lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java Tue Jul 16 17:00:05 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.queryparser.an
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -165,7 +164,7 @@ public class AnalyzingQueryParser extend
String analyzed = null;
TokenStream stream = null;
try{
- stream = getAnalyzer().tokenStream(field, new StringReader(chunk));
+ stream = getAnalyzer().tokenStream(field, chunk);
stream.reset();
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
// get first and hopefully only output token
Modified: lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/lucene3069/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Tue Jul 16 17:00:05 2013
@@ -500,7 +500,7 @@ public abstract class QueryParserBase im
TokenStream source;
try {
- source = analyzer.tokenStream(field, new StringReader(queryText));
+ source = analyzer.tokenStream(field, queryText);
source.reset();
} catch (IOException e) {
ParseException p = new ParseException("Unable to initialize TokenStream to analyze query text");
@@ -844,7 +844,7 @@ public abstract class QueryParserBase im
if (analyzerIn == null) analyzerIn = analyzer;
try {
- source = analyzerIn.tokenStream(field, new StringReader(part));
+ source = analyzerIn.tokenStream(field, part);
source.reset();
} catch (IOException e) {
throw new RuntimeException("Unable to initialize TokenStream to analyze multiTerm term: " + part, e);