You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/11 14:19:39 UTC
svn commit: r1512909 [17/38] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-too...
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeAccumulator.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeAccumulator.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeAccumulator.java Sun Aug 11 12:19:13 2013
@@ -17,6 +17,7 @@ package org.apache.lucene.facet.range;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -37,13 +38,15 @@ import org.apache.lucene.facet.params.Fa
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.search.CountFacetRequest;
import org.apache.lucene.facet.search.DrillDownQuery;
-import org.apache.lucene.facet.search.DrillSideways.DrillSidewaysResult;
import org.apache.lucene.facet.search.DrillSideways;
+import org.apache.lucene.facet.search.DrillSideways.DrillSidewaysResult;
import org.apache.lucene.facet.search.FacetRequest;
import org.apache.lucene.facet.search.FacetResult;
import org.apache.lucene.facet.search.FacetResultNode;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.FacetsCollector;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetFields;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@@ -74,15 +77,12 @@ public class TestRangeAccumulator extend
IndexReader r = w.getReader();
w.close();
- FacetSearchParams fsp = new FacetSearchParams(
- new RangeFacetRequest<LongRange>("field",
- new LongRange("less than 10", 0L, true, 10L, false),
- new LongRange("less than or equal to 10", 0L, true, 10L, true),
- new LongRange("over 90", 90L, false, 100L, false),
- new LongRange("90 or above", 90L, true, 100L, false),
- new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, false)));
-
- RangeAccumulator a = new RangeAccumulator(fsp, r);
+ RangeAccumulator a = new RangeAccumulator(new RangeFacetRequest<LongRange>("field",
+ new LongRange("less than 10", 0L, true, 10L, false),
+ new LongRange("less than or equal to 10", 0L, true, 10L, true),
+ new LongRange("over 90", 90L, false, 100L, false),
+ new LongRange("90 or above", 90L, true, 100L, false),
+ new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, false)));
FacetsCollector fc = FacetsCollector.create(a);
@@ -97,15 +97,15 @@ public class TestRangeAccumulator extend
}
/** Tests single request that mixes Range and non-Range
- * faceting, with DrillSideways. */
- public void testMixedRangeAndNonRange() throws Exception {
+ * faceting, with DrillSideways and taxonomy. */
+ public void testMixedRangeAndNonRangeTaxonomy() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Directory td = newDirectory();
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, IndexWriterConfig.OpenMode.CREATE);
FacetFields ff = new FacetFields(tw);
- for(long l=0;l<100;l++) {
+ for (long l = 0; l < 100; l++) {
Document doc = new Document();
// For computing range facet counts:
doc.add(new NumericDocValuesField("field", l));
@@ -122,7 +122,7 @@ public class TestRangeAccumulator extend
w.addDocument(doc);
}
- IndexReader r = w.getReader();
+ final IndexReader r = w.getReader();
w.close();
final TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
@@ -130,32 +130,32 @@ public class TestRangeAccumulator extend
IndexSearcher s = newSearcher(r);
- final FacetSearchParams fsp = new FacetSearchParams(
- new CountFacetRequest(new CategoryPath("dim"), 2),
- new RangeFacetRequest<LongRange>("field",
- new LongRange("less than 10", 0L, true, 10L, false),
- new LongRange("less than or equal to 10", 0L, true, 10L, true),
- new LongRange("over 90", 90L, false, 100L, false),
- new LongRange("90 or above", 90L, true, 100L, false),
- new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, false)));
-
+ final CountFacetRequest countRequest = new CountFacetRequest(new CategoryPath("dim"), 2);
+ final RangeFacetRequest<LongRange> rangeRequest = new RangeFacetRequest<LongRange>("field",
+ new LongRange("less than 10", 0L, true, 10L, false),
+ new LongRange("less than or equal to 10", 0L, true, 10L, true),
+ new LongRange("over 90", 90L, false, 100L, false),
+ new LongRange("90 or above", 90L, true, 100L, false),
+ new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, false));
+ FacetSearchParams fsp = new FacetSearchParams(countRequest, rangeRequest);
+
final Set<String> dimSeen = new HashSet<String>();
DrillSideways ds = new DrillSideways(s, tr) {
@Override
protected FacetsAccumulator getDrillDownAccumulator(FacetSearchParams fsp) {
checkSeen(fsp);
- return RangeFacetsAccumulatorWrapper.create(fsp, searcher.getIndexReader(), tr);
+ return FacetsAccumulator.create(fsp, r, tr, null);
}
@Override
protected FacetsAccumulator getDrillSidewaysAccumulator(String dim, FacetSearchParams fsp) {
checkSeen(fsp);
- return RangeFacetsAccumulatorWrapper.create(fsp, searcher.getIndexReader(), tr);
+ return FacetsAccumulator.create(fsp, r, tr, null);
}
private void checkSeen(FacetSearchParams fsp) {
- // Each dim should should up only once, across
+ // Each dim should up only once, across
// both drillDown and drillSideways requests:
for(FacetRequest fr : fsp.facetRequests) {
String dim = fr.categoryPath.components[0];
@@ -204,6 +204,111 @@ public class TestRangeAccumulator extend
IOUtils.close(tr, td, r, d);
}
+ /** Tests single request that mixes Range and non-Range
+ * faceting, with DrillSideways and SortedSet. */
+ public void testMixedRangeAndNonRangeSortedSet() throws Exception {
+ assumeTrue("Test requires SortedSetDV support", defaultCodecSupportsSortedSet());
+ Directory d = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), d);
+ SortedSetDocValuesFacetFields ff = new SortedSetDocValuesFacetFields();
+
+ for (long l = 0; l < 100; l++) {
+ Document doc = new Document();
+ // For computing range facet counts:
+ doc.add(new NumericDocValuesField("field", l));
+ // For drill down by numeric range:
+ doc.add(new LongField("field", l, Field.Store.NO));
+
+ CategoryPath cp;
+ if ((l&3) == 0) {
+ cp = new CategoryPath("dim", "a");
+ } else {
+ cp = new CategoryPath("dim", "b");
+ }
+ ff.addFields(doc, Collections.singletonList(cp));
+ w.addDocument(doc);
+ }
+
+ final IndexReader r = w.getReader();
+ w.close();
+
+ IndexSearcher s = newSearcher(r);
+ final SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(s.getIndexReader());
+
+ final CountFacetRequest countRequest = new CountFacetRequest(new CategoryPath("dim"), 2);
+ final RangeFacetRequest<LongRange> rangeRequest = new RangeFacetRequest<LongRange>("field",
+ new LongRange("less than 10", 0L, true, 10L, false),
+ new LongRange("less than or equal to 10", 0L, true, 10L, true),
+ new LongRange("over 90", 90L, false, 100L, false),
+ new LongRange("90 or above", 90L, true, 100L, false),
+ new LongRange("over 1000", 1000L, false, Long.MAX_VALUE, false));
+ FacetSearchParams fsp = new FacetSearchParams(countRequest, rangeRequest);
+
+ final Set<String> dimSeen = new HashSet<String>();
+
+ DrillSideways ds = new DrillSideways(s, state) {
+ @Override
+ protected FacetsAccumulator getDrillDownAccumulator(FacetSearchParams fsp) throws IOException {
+ checkSeen(fsp);
+ return FacetsAccumulator.create(fsp, state, null);
+ }
+
+ @Override
+ protected FacetsAccumulator getDrillSidewaysAccumulator(String dim, FacetSearchParams fsp) throws IOException {
+ checkSeen(fsp);
+ return FacetsAccumulator.create(fsp, state, null);
+ }
+
+ private void checkSeen(FacetSearchParams fsp) {
+ // Each dim should up only once, across
+ // both drillDown and drillSideways requests:
+ for(FacetRequest fr : fsp.facetRequests) {
+ String dim = fr.categoryPath.components[0];
+ assertFalse("dim " + dim + " already seen", dimSeen.contains(dim));
+ dimSeen.add(dim);
+ }
+ }
+
+ @Override
+ protected boolean scoreSubDocsAtOnce() {
+ return random().nextBoolean();
+ }
+ };
+
+ // First search, no drill downs:
+ DrillDownQuery ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT, new MatchAllDocsQuery());
+ DrillSidewaysResult dsr = ds.search(null, ddq, 10, fsp);
+
+ assertEquals(100, dsr.hits.totalHits);
+ assertEquals(2, dsr.facetResults.size());
+ assertEquals("dim (0)\n b (75)\n a (25)\n", FacetTestUtils.toSimpleString(dsr.facetResults.get(0)));
+ assertEquals("field (0)\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", FacetTestUtils.toSimpleString(dsr.facetResults.get(1)));
+
+ // Second search, drill down on dim=b:
+ ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT, new MatchAllDocsQuery());
+ ddq.add(new CategoryPath("dim", "b"));
+ dimSeen.clear();
+ dsr = ds.search(null, ddq, 10, fsp);
+
+ assertEquals(75, dsr.hits.totalHits);
+ assertEquals(2, dsr.facetResults.size());
+ assertEquals("dim (0)\n b (75)\n a (25)\n", FacetTestUtils.toSimpleString(dsr.facetResults.get(0)));
+ assertEquals("field (0)\n less than 10 (7)\n less than or equal to 10 (8)\n over 90 (7)\n 90 or above (8)\n over 1000 (0)\n", FacetTestUtils.toSimpleString(dsr.facetResults.get(1)));
+
+ // Third search, drill down on "less than or equal to 10":
+ ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT, new MatchAllDocsQuery());
+ ddq.add("field", NumericRangeQuery.newLongRange("field", 0L, 10L, true, true));
+ dimSeen.clear();
+ dsr = ds.search(null, ddq, 10, fsp);
+
+ assertEquals(11, dsr.hits.totalHits);
+ assertEquals(2, dsr.facetResults.size());
+ assertEquals("dim (0)\n b (8)\n a (3)\n", FacetTestUtils.toSimpleString(dsr.facetResults.get(0)));
+ assertEquals("field (0)\n less than 10 (10)\n less than or equal to 10 (11)\n over 90 (9)\n 90 or above (10)\n over 1000 (0)\n", FacetTestUtils.toSimpleString(dsr.facetResults.get(1)));
+
+ IOUtils.close(r, d);
+ }
+
public void testBasicDouble() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
@@ -211,22 +316,19 @@ public class TestRangeAccumulator extend
DoubleDocValuesField field = new DoubleDocValuesField("field", 0.0);
doc.add(field);
for(long l=0;l<100;l++) {
- field.setDoubleValue((double) l);
+ field.setDoubleValue(l);
w.addDocument(doc);
}
IndexReader r = w.getReader();
w.close();
- FacetSearchParams fsp = new FacetSearchParams(
- new RangeFacetRequest<DoubleRange>("field",
- new DoubleRange("less than 10", 0.0, true, 10.0, false),
- new DoubleRange("less than or equal to 10", 0.0, true, 10.0, true),
- new DoubleRange("over 90", 90.0, false, 100.0, false),
- new DoubleRange("90 or above", 90.0, true, 100.0, false),
- new DoubleRange("over 1000", 1000.0, false, Double.POSITIVE_INFINITY, false)));
-
- RangeAccumulator a = new RangeAccumulator(fsp, r);
+ RangeAccumulator a = new RangeAccumulator(new RangeFacetRequest<DoubleRange>("field",
+ new DoubleRange("less than 10", 0.0, true, 10.0, false),
+ new DoubleRange("less than or equal to 10", 0.0, true, 10.0, true),
+ new DoubleRange("over 90", 90.0, false, 100.0, false),
+ new DoubleRange("90 or above", 90.0, true, 100.0, false),
+ new DoubleRange("over 1000", 1000.0, false, Double.POSITIVE_INFINITY, false)));
FacetsCollector fc = FacetsCollector.create(a);
@@ -247,22 +349,19 @@ public class TestRangeAccumulator extend
FloatDocValuesField field = new FloatDocValuesField("field", 0.0f);
doc.add(field);
for(long l=0;l<100;l++) {
- field.setFloatValue((float) l);
+ field.setFloatValue(l);
w.addDocument(doc);
}
IndexReader r = w.getReader();
w.close();
- FacetSearchParams fsp = new FacetSearchParams(
- new RangeFacetRequest<FloatRange>("field",
- new FloatRange("less than 10", 0.0f, true, 10.0f, false),
- new FloatRange("less than or equal to 10", 0.0f, true, 10.0f, true),
- new FloatRange("over 90", 90.0f, false, 100.0f, false),
- new FloatRange("90 or above", 90.0f, true, 100.0f, false),
- new FloatRange("over 1000", 1000.0f, false, Float.POSITIVE_INFINITY, false)));
-
- RangeAccumulator a = new RangeAccumulator(fsp, r);
+ RangeAccumulator a = new RangeAccumulator(new RangeFacetRequest<FloatRange>("field",
+ new FloatRange("less than 10", 0.0f, true, 10.0f, false),
+ new FloatRange("less than or equal to 10", 0.0f, true, 10.0f, true),
+ new FloatRange("over 90", 90.0f, false, 100.0f, false),
+ new FloatRange("90 or above", 90.0f, true, 100.0f, false),
+ new FloatRange("over 1000", 1000.0f, false, Float.POSITIVE_INFINITY, false)));
FacetsCollector fc = FacetsCollector.create(a);
@@ -335,8 +434,7 @@ public class TestRangeAccumulator extend
}
}
- FacetSearchParams fsp = new FacetSearchParams(new RangeFacetRequest<LongRange>("field", ranges));
- FacetsCollector fc = FacetsCollector.create(new RangeAccumulator(fsp, r));
+ FacetsCollector fc = FacetsCollector.create(new RangeAccumulator(new RangeFacetRequest<LongRange>("field", ranges)));
s.search(new MatchAllDocsQuery(), fc);
List<FacetResult> results = fc.getFacetResults();
assertEquals(1, results.size());
@@ -350,7 +448,7 @@ public class TestRangeAccumulator extend
assertEquals("field/r" + rangeID, subNode.label.toString('/'));
assertEquals(expectedCounts[rangeID], (int) subNode.value);
- LongRange range = (LongRange) ((RangeFacetRequest) results.get(0).getFacetRequest()).ranges[rangeID];
+ LongRange range = (LongRange) ((RangeFacetRequest<?>) results.get(0).getFacetRequest()).ranges[rangeID];
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT);
@@ -422,8 +520,7 @@ public class TestRangeAccumulator extend
}
}
- FacetSearchParams fsp = new FacetSearchParams(new RangeFacetRequest<FloatRange>("field", ranges));
- FacetsCollector fc = FacetsCollector.create(new RangeAccumulator(fsp, r));
+ FacetsCollector fc = FacetsCollector.create(new RangeAccumulator(new RangeFacetRequest<FloatRange>("field", ranges)));
s.search(new MatchAllDocsQuery(), fc);
List<FacetResult> results = fc.getFacetResults();
assertEquals(1, results.size());
@@ -437,7 +534,7 @@ public class TestRangeAccumulator extend
assertEquals("field/r" + rangeID, subNode.label.toString('/'));
assertEquals(expectedCounts[rangeID], (int) subNode.value);
- FloatRange range = (FloatRange) ((RangeFacetRequest) results.get(0).getFacetRequest()).ranges[rangeID];
+ FloatRange range = (FloatRange) ((RangeFacetRequest<?>) results.get(0).getFacetRequest()).ranges[rangeID];
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT);
@@ -509,8 +606,7 @@ public class TestRangeAccumulator extend
}
}
- FacetSearchParams fsp = new FacetSearchParams(new RangeFacetRequest<DoubleRange>("field", ranges));
- FacetsCollector fc = FacetsCollector.create(new RangeAccumulator(fsp, r));
+ FacetsCollector fc = FacetsCollector.create(new RangeAccumulator(new RangeFacetRequest<DoubleRange>("field", ranges)));
s.search(new MatchAllDocsQuery(), fc);
List<FacetResult> results = fc.getFacetResults();
assertEquals(1, results.size());
@@ -524,7 +620,7 @@ public class TestRangeAccumulator extend
assertEquals("field/r" + rangeID, subNode.label.toString('/'));
assertEquals(expectedCounts[rangeID], (int) subNode.value);
- DoubleRange range = (DoubleRange) ((RangeFacetRequest) results.get(0).getFacetRequest()).ranges[rangeID];
+ DoubleRange range = (DoubleRange) ((RangeFacetRequest<?>) results.get(0).getFacetRequest()).ranges[rangeID];
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT);
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/BaseSampleTestTopK.java Sun Aug 11 12:19:13 2013
@@ -3,18 +3,14 @@ package org.apache.lucene.facet.sampling
import java.util.List;
import java.util.Random;
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
-import org.apache.lucene.facet.sampling.RandomSampler;
-import org.apache.lucene.facet.sampling.RepeatableSampler;
-import org.apache.lucene.facet.sampling.Sampler;
-import org.apache.lucene.facet.sampling.SamplingParams;
import org.apache.lucene.facet.search.BaseTestTopK;
import org.apache.lucene.facet.search.FacetRequest;
+import org.apache.lucene.facet.search.FacetRequest.ResultMode;
import org.apache.lucene.facet.search.FacetResult;
import org.apache.lucene.facet.search.FacetsCollector;
-import org.apache.lucene.facet.search.StandardFacetsAccumulator;
-import org.apache.lucene.facet.search.FacetRequest.ResultMode;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@@ -60,7 +56,7 @@ public abstract class BaseSampleTestTopK
return res;
}
- protected abstract StandardFacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
+ protected abstract OldFacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
IndexReader indexReader, FacetSearchParams searchParams);
/**
@@ -94,7 +90,7 @@ public abstract class BaseSampleTestTopK
for (int nTrial = 0; nTrial < RETRIES; nTrial++) {
try {
// complement with sampling!
- final Sampler sampler = createSampler(nTrial, useRandomSampler);
+ final Sampler sampler = createSampler(nTrial, useRandomSampler, samplingSearchParams);
assertSampling(expectedResults, q, sampler, samplingSearchParams, false);
assertSampling(expectedResults, q, sampler, samplingSearchParams, true);
@@ -123,19 +119,25 @@ public abstract class BaseSampleTestTopK
private FacetsCollector samplingCollector(final boolean complement, final Sampler sampler,
FacetSearchParams samplingSearchParams) {
- StandardFacetsAccumulator sfa = getSamplingAccumulator(sampler, taxoReader, indexReader, samplingSearchParams);
- sfa.setComplementThreshold(complement ? StandardFacetsAccumulator.FORCE_COMPLEMENT : StandardFacetsAccumulator.DISABLE_COMPLEMENT);
+ OldFacetsAccumulator sfa = getSamplingAccumulator(sampler, taxoReader, indexReader, samplingSearchParams);
+ sfa.setComplementThreshold(complement ? OldFacetsAccumulator.FORCE_COMPLEMENT : OldFacetsAccumulator.DISABLE_COMPLEMENT);
return FacetsCollector.create(sfa);
}
- private Sampler createSampler(int nTrial, boolean useRandomSampler) {
+ private Sampler createSampler(int nTrial, boolean useRandomSampler, FacetSearchParams sParams) {
SamplingParams samplingParams = new SamplingParams();
+ /*
+ * Set sampling to Exact fixing with TakmiSampleFixer as it is not easy to
+ * validate results with amortized results.
+ */
+ samplingParams.setSampleFixer(new TakmiSampleFixer(indexReader, taxoReader, sParams));
+
final double retryFactor = Math.pow(1.01, nTrial);
+ samplingParams.setOversampleFactor(5.0 * retryFactor); // Oversampling
samplingParams.setSampleRatio(0.8 * retryFactor);
samplingParams.setMinSampleSize((int) (100 * retryFactor));
samplingParams.setMaxSampleSize((int) (10000 * retryFactor));
- samplingParams.setOversampleFactor(5.0 * retryFactor);
samplingParams.setSamplingThreshold(11000); //force sampling
Sampler sampler = useRandomSampler ?
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/OversampleWithDepthTest.java Sun Aug 11 12:19:13 2013
@@ -6,19 +6,15 @@ import java.util.Collections;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
-import org.apache.lucene.facet.sampling.RandomSampler;
-import org.apache.lucene.facet.sampling.Sampler;
-import org.apache.lucene.facet.sampling.SamplingAccumulator;
-import org.apache.lucene.facet.sampling.SamplingParams;
import org.apache.lucene.facet.search.CountFacetRequest;
import org.apache.lucene.facet.search.FacetRequest;
+import org.apache.lucene.facet.search.FacetRequest.ResultMode;
import org.apache.lucene.facet.search.FacetResult;
import org.apache.lucene.facet.search.FacetResultNode;
import org.apache.lucene.facet.search.FacetsCollector;
-import org.apache.lucene.facet.search.StandardFacetsAccumulator;
-import org.apache.lucene.facet.search.FacetRequest.ResultMode;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
@@ -116,7 +112,7 @@ public class OversampleWithDepthTest ext
final SamplingParams params) throws IOException {
// a FacetsCollector with a sampling accumulator
Sampler sampler = new RandomSampler(params, random());
- StandardFacetsAccumulator sfa = new SamplingAccumulator(sampler, fsp, r, tr);
+ OldFacetsAccumulator sfa = new SamplingAccumulator(sampler, fsp, r, tr);
FacetsCollector fcWithSampling = FacetsCollector.create(sfa);
IndexSearcher s = newSearcher(r);
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingAccumulatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingAccumulatorTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingAccumulatorTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingAccumulatorTest.java Sun Aug 11 12:19:13 2013
@@ -1,9 +1,7 @@
package org.apache.lucene.facet.sampling;
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.FacetSearchParams;
-import org.apache.lucene.facet.sampling.Sampler;
-import org.apache.lucene.facet.sampling.SamplingAccumulator;
-import org.apache.lucene.facet.search.StandardFacetsAccumulator;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.LuceneTestCase.Slow;
@@ -29,7 +27,7 @@ import org.apache.lucene.util.LuceneTest
public class SamplingAccumulatorTest extends BaseSampleTestTopK {
@Override
- protected StandardFacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
+ protected OldFacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
IndexReader indexReader, FacetSearchParams searchParams) {
return new SamplingAccumulator(sampler, searchParams, indexReader, taxoReader);
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingWrapperTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingWrapperTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sampling/SamplingWrapperTest.java Sun Aug 11 12:19:13 2013
@@ -1,13 +1,10 @@
package org.apache.lucene.facet.sampling;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.util.LuceneTestCase.Slow;
-
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.FacetSearchParams;
-import org.apache.lucene.facet.sampling.Sampler;
-import org.apache.lucene.facet.sampling.SamplingWrapper;
-import org.apache.lucene.facet.search.StandardFacetsAccumulator;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.LuceneTestCase.Slow;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -30,9 +27,9 @@ import org.apache.lucene.facet.taxonomy.
public class SamplingWrapperTest extends BaseSampleTestTopK {
@Override
- protected StandardFacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
+ protected OldFacetsAccumulator getSamplingAccumulator(Sampler sampler, TaxonomyReader taxoReader,
IndexReader indexReader, FacetSearchParams searchParams) {
- return new SamplingWrapper(new StandardFacetsAccumulator(searchParams, indexReader, taxoReader), sampler);
+ return new SamplingWrapper(new OldFacetsAccumulator(searchParams, indexReader, taxoReader), sampler);
}
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsAggregatorTest.java Sun Aug 11 12:19:13 2013
@@ -269,7 +269,7 @@ public class CountingFacetsAggregatorTes
IOUtils.close(indexWriter, taxoWriter);
}
- private FacetsAccumulator randomAccumulator(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
+ private TaxonomyFacetsAccumulator randomAccumulator(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
final FacetsAggregator aggregator;
double val = random().nextDouble();
if (val < 0.6) {
@@ -279,7 +279,7 @@ public class CountingFacetsAggregatorTes
} else {
aggregator = new CachedOrdsCountingFacetsAggregator();
}
- return new FacetsAccumulator(fsp, indexReader, taxoReader) {
+ return new TaxonomyFacetsAccumulator(fsp, indexReader, taxoReader) {
@Override
public FacetsAggregator getAggregator() {
return aggregator;
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/FacetRequestTest.java Sun Aug 11 12:19:13 2013
@@ -23,7 +23,7 @@ import org.junit.Test;
*/
public class FacetRequestTest extends FacetTestCase {
-
+
@Test(expected=IllegalArgumentException.class)
public void testIllegalNumResults() throws Exception {
assertNotNull(new CountFacetRequest(new CategoryPath("a", "b"), 0));
@@ -33,7 +33,7 @@ public class FacetRequestTest extends Fa
public void testIllegalCategoryPath() throws Exception {
assertNotNull(new CountFacetRequest(null, 1));
}
-
+
@Test
public void testHashAndEquals() {
CountFacetRequest fr1 = new CountFacetRequest(new CategoryPath("a"), 8);
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java Sun Aug 11 12:19:13 2013
@@ -28,16 +28,19 @@ import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.FacetTestUtils;
+import org.apache.lucene.facet.codecs.facet42.Facet42DocValuesFormat;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.search.DrillSideways.DrillSidewaysResult;
-import org.apache.lucene.facet.sortedset.SortedSetDocValuesAccumulator;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetFields;
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.CategoryPath;
@@ -59,15 +62,18 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util._TestUtil;
+import org.junit.Test;
public class TestDrillSideways extends FacetTestCase {
@@ -329,6 +335,8 @@ public class TestDrillSideways extends F
String id;
String contentToken;
+ public Doc() {}
+
// -1 if the doc is missing this dim, else the index
// -into the values for this dim:
int[] dims;
@@ -426,6 +434,16 @@ public class TestDrillSideways extends F
boolean canUseDV = defaultCodecSupportsSortedSet();
+ // TestRuleSetupAndRestoreClassEnv can sometimes
+ // randomly pick the non-general Facet42DocValuesFormat:
+ DocValuesFormat dvf = Codec.getDefault().docValuesFormat();
+ if (dvf instanceof PerFieldDocValuesFormat) {
+ dvf = ((PerFieldDocValuesFormat) dvf).getDocValuesFormatForField("$facets");
+ }
+ if (dvf instanceof Facet42DocValuesFormat) {
+ canUseDV = false;
+ }
+
while (aChance == 0.0) {
aChance = random().nextDouble();
}
@@ -594,8 +612,9 @@ public class TestDrillSideways extends F
w.close();
final SortedSetDocValuesReaderState sortedSetDVState;
+ IndexSearcher s = newSearcher(r);
if (doUseDV) {
- sortedSetDVState = new SortedSetDocValuesReaderState(r);
+ sortedSetDVState = new SortedSetDocValuesReaderState(s.getIndexReader());
} else {
sortedSetDVState = null;
}
@@ -608,8 +627,6 @@ public class TestDrillSideways extends F
TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
tw.close();
- IndexSearcher s = newSearcher(r);
-
int numIters = atLeast(10);
for(int iter=0;iter<numIters;iter++) {
@@ -774,17 +791,7 @@ public class TestDrillSideways extends F
Sort sort = new Sort(new SortField("id", SortField.Type.STRING));
DrillSideways ds;
if (doUseDV) {
- ds = new DrillSideways(s, null) {
- @Override
- protected FacetsAccumulator getDrillDownAccumulator(FacetSearchParams fsp) throws IOException {
- return new SortedSetDocValuesAccumulator(fsp, sortedSetDVState);
- }
-
- @Override
- protected FacetsAccumulator getDrillSidewaysAccumulator(String dim, FacetSearchParams fsp) throws IOException {
- return new SortedSetDocValuesAccumulator(fsp, sortedSetDVState);
- }
- };
+ ds = new DrillSideways(s, sortedSetDVState);
} else {
ds = new DrillSideways(s, tr);
}
@@ -865,6 +872,7 @@ public class TestDrillSideways extends F
List<Doc> hits;
int[][] counts;
int[] uniqueCounts;
+ public SimpleFacetResult() {}
}
private int[] getTopNOrds(final int[] counts, final String[] values, int topN) {
@@ -1144,5 +1152,33 @@ public class TestDrillSideways extends F
}
return b.toString();
}
+
+ @Test
+ public void testEmptyIndex() throws Exception {
+ // LUCENE-5045: make sure DrillSideways works with an empty index
+ Directory dir = newDirectory();
+ Directory taxoDir = newDirectory();
+ writer = new RandomIndexWriter(random(), dir);
+ taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
+ IndexSearcher searcher = newSearcher(writer.getReader());
+ writer.close();
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
+ taxoWriter.close();
+
+ // Count "Author"
+ FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(new CategoryPath("Author"), 10));
+
+ DrillSideways ds = new DrillSideways(searcher, taxoReader);
+ DrillDownQuery ddq = new DrillDownQuery(fsp.indexingParams, new MatchAllDocsQuery());
+ ddq.add(new CategoryPath("Author", "Lisa"));
+
+ DrillSidewaysResult r = ds.search(null, ddq, 10, fsp); // this used to fail on IllegalArgEx
+ assertEquals(0, r.hits.totalHits);
+
+ r = ds.search(ddq, null, null, 10, new Sort(new SortField("foo", Type.INT)), false, false, fsp); // this used to fail on IllegalArgEx
+ assertEquals(0, r.hits.totalHits);
+
+ IOUtils.close(searcher.getIndexReader(), taxoReader, dir, taxoDir);
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsCollector.java Sun Aug 11 12:19:13 2013
@@ -3,9 +3,7 @@ package org.apache.lucene.facet.search;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -13,12 +11,26 @@ import org.apache.lucene.document.Field.
import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.old.AdaptiveFacetsAccumulator;
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.CategoryListParams;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.range.LongRange;
+import org.apache.lucene.facet.range.RangeAccumulator;
+import org.apache.lucene.facet.range.RangeFacetRequest;
+import org.apache.lucene.facet.sampling.RandomSampler;
+import org.apache.lucene.facet.sampling.Sampler;
+import org.apache.lucene.facet.sampling.SamplingAccumulator;
+import org.apache.lucene.facet.sampling.SamplingParams;
+import org.apache.lucene.facet.sampling.SamplingWrapper;
+import org.apache.lucene.facet.sampling.TakmiSampleFixer;
import org.apache.lucene.facet.search.FacetRequest.ResultMode;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesAccumulator;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
@@ -78,7 +90,7 @@ public class TestFacetsCollector extends
DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);
FacetSearchParams sParams = new FacetSearchParams(new SumScoreFacetRequest(new CategoryPath("a"), 10));
- FacetsAccumulator fa = new FacetsAccumulator(sParams, r, taxo) {
+ TaxonomyFacetsAccumulator fa = new TaxonomyFacetsAccumulator(sParams, r, taxo) {
@Override
public FacetsAggregator getAggregator() {
return new SumScoreFacetsAggregator();
@@ -169,18 +181,7 @@ public class TestFacetsCollector extends
new CountFacetRequest(new CategoryPath("a"), 10),
new SumScoreFacetRequest(new CategoryPath("b"), 10));
- Map<CategoryListParams,FacetsAggregator> aggregators = new HashMap<CategoryListParams,FacetsAggregator>();
- aggregators.put(fip.getCategoryListParams(new CategoryPath("a")), new FastCountingFacetsAggregator());
- aggregators.put(fip.getCategoryListParams(new CategoryPath("b")), new SumScoreFacetsAggregator());
- final FacetsAggregator aggregator = new PerCategoryListAggregator(aggregators, fip);
- FacetsAccumulator fa = new FacetsAccumulator(sParams, r, taxo) {
- @Override
- public FacetsAggregator getAggregator() {
- return aggregator;
- }
- };
-
- FacetsCollector fc = FacetsCollector.create(fa);
+ FacetsCollector fc = FacetsCollector.create(sParams, r, taxo);
TopScoreDocCollector topDocs = TopScoreDocCollector.create(10, false);
newSearcher(r).search(new MatchAllDocsQuery(), MultiCollector.wrap(fc, topDocs));
@@ -219,7 +220,7 @@ public class TestFacetsCollector extends
FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(CategoryPath.EMPTY, 10));
- final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ final TaxonomyFacetsAccumulator fa = random().nextBoolean() ? new TaxonomyFacetsAccumulator(fsp, r, taxo) : new OldFacetsAccumulator(fsp, r, taxo);
FacetsCollector fc = FacetsCollector.create(fa);
newSearcher(r).search(new MatchAllDocsQuery(), fc);
@@ -253,7 +254,7 @@ public class TestFacetsCollector extends
FacetSearchParams fsp = new FacetSearchParams(
new CountFacetRequest(new CategoryPath("a"), 10),
new CountFacetRequest(new CategoryPath("b"), 10));
- final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ final TaxonomyFacetsAccumulator fa = random().nextBoolean() ? new TaxonomyFacetsAccumulator(fsp, r, taxo) : new OldFacetsAccumulator(fsp, r, taxo);
final FacetsCollector fc = FacetsCollector.create(fa);
newSearcher(r).search(new MatchAllDocsQuery(), fc);
@@ -285,7 +286,7 @@ public class TestFacetsCollector extends
FacetSearchParams fsp = new FacetSearchParams(
new CountFacetRequest(new CategoryPath("a"), 10),
new CountFacetRequest(new CategoryPath("b"), 10));
- final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ final TaxonomyFacetsAccumulator fa = random().nextBoolean() ? new TaxonomyFacetsAccumulator(fsp, r, taxo) : new OldFacetsAccumulator(fsp, r, taxo);
final FacetsCollector fc = FacetsCollector.create(fa);
// this should populate the cached results, but doing search should clear the cache
fc.getFacetResults();
@@ -326,7 +327,7 @@ public class TestFacetsCollector extends
// assert IntFacetResultHandler
FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(new CategoryPath("a"), 10));
- FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ TaxonomyFacetsAccumulator fa = random().nextBoolean() ? new TaxonomyFacetsAccumulator(fsp, r, taxo) : new OldFacetsAccumulator(fsp, r, taxo);
FacetsCollector fc = FacetsCollector.create(fa);
newSearcher(r).search(new MatchAllDocsQuery(), fc);
assertTrue("invalid ordinal for child node: 0", 0 != fc.getFacetResults().get(0).getFacetResultNode().subResults.get(0).ordinal);
@@ -334,14 +335,14 @@ public class TestFacetsCollector extends
// assert IntFacetResultHandler
fsp = new FacetSearchParams(new SumScoreFacetRequest(new CategoryPath("a"), 10));
if (random().nextBoolean()) {
- fa = new FacetsAccumulator(fsp, r, taxo) {
+ fa = new TaxonomyFacetsAccumulator(fsp, r, taxo) {
@Override
public FacetsAggregator getAggregator() {
return new SumScoreFacetsAggregator();
}
};
} else {
- fa = new StandardFacetsAccumulator(fsp, r, taxo);
+ fa = new OldFacetsAccumulator(fsp, r, taxo);
}
fc = FacetsCollector.create(fa);
newSearcher(r).search(new MatchAllDocsQuery(), fc);
@@ -375,7 +376,7 @@ public class TestFacetsCollector extends
CountFacetRequest cfr = new CountFacetRequest(new CategoryPath("a"), 2);
cfr.setResultMode(random().nextBoolean() ? ResultMode.GLOBAL_FLAT : ResultMode.PER_NODE_IN_TREE);
FacetSearchParams fsp = new FacetSearchParams(cfr);
- final FacetsAccumulator fa = random().nextBoolean() ? new FacetsAccumulator(fsp, r, taxo) : new StandardFacetsAccumulator(fsp, r, taxo);
+ final TaxonomyFacetsAccumulator fa = random().nextBoolean() ? new TaxonomyFacetsAccumulator(fsp, r, taxo) : new OldFacetsAccumulator(fsp, r, taxo);
FacetsCollector fc = FacetsCollector.create(fa);
newSearcher(r).search(new MatchAllDocsQuery(), fc);
@@ -384,5 +385,71 @@ public class TestFacetsCollector extends
IOUtils.close(taxo, taxoDir, r, indexDir);
}
-
+
+ @Test
+ public void testLabeling() throws Exception {
+ Directory indexDir = newDirectory(), taxoDir = newDirectory();
+
+ // create the index
+ IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+ FacetFields facetFields = new FacetFields(taxoWriter);
+ Document doc = new Document();
+ facetFields.addFields(doc, Arrays.asList(new CategoryPath("A/1", '/')));
+ indexWriter.addDocument(doc);
+ IOUtils.close(indexWriter, taxoWriter);
+
+ DirectoryReader indexReader = DirectoryReader.open(indexDir);
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = new IndexSearcher(indexReader);
+ // ask to count a non-existing category to test labeling
+ FacetSearchParams fsp = new FacetSearchParams(new CountFacetRequest(new CategoryPath("B"), 5));
+
+ final SamplingParams sampleParams = new SamplingParams();
+ sampleParams.setMaxSampleSize(100);
+ sampleParams.setMinSampleSize(100);
+ sampleParams.setSamplingThreshold(100);
+ sampleParams.setOversampleFactor(1.0d);
+ if (random().nextBoolean()) {
+ sampleParams.setSampleFixer(new TakmiSampleFixer(indexReader, taxoReader, fsp));
+ }
+ final Sampler sampler = new RandomSampler(sampleParams, random());
+
+ TaxonomyFacetsAccumulator[] accumulators = new TaxonomyFacetsAccumulator[] {
+ new TaxonomyFacetsAccumulator(fsp, indexReader, taxoReader),
+ new OldFacetsAccumulator(fsp, indexReader, taxoReader),
+ new SamplingAccumulator(sampler, fsp, indexReader, taxoReader),
+ new AdaptiveFacetsAccumulator(fsp, indexReader, taxoReader),
+ new SamplingWrapper(new OldFacetsAccumulator(fsp, indexReader, taxoReader), sampler)
+ };
+
+ for (TaxonomyFacetsAccumulator fa : accumulators) {
+ FacetsCollector fc = FacetsCollector.create(fa);
+ searcher.search(new MatchAllDocsQuery(), fc);
+ List<FacetResult> facetResults = fc.getFacetResults();
+ assertNotNull(facetResults);
+ assertEquals("incorrect label returned for " + fa, fsp.facetRequests.get(0).categoryPath, facetResults.get(0).getFacetResultNode().label);
+ }
+
+ try {
+ // SortedSetDocValuesAccumulator cannot even be created in such state
+ assertNull(new SortedSetDocValuesAccumulator(new SortedSetDocValuesReaderState(indexReader), fsp));
+ // if this ever changes, make sure FacetResultNode is labeled correctly
+ fail("should not have succeeded to execute a request over a category which wasn't indexed as SortedSetDVField");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ RangeAccumulator ra = new RangeAccumulator(new RangeFacetRequest<LongRange>("f", new LongRange("grr", 0, true, 1, true)));
+ FacetsCollector fc = FacetsCollector.create(ra);
+ searcher.search(new MatchAllDocsQuery(), fc);
+ List<FacetResult> facetResults = fc.getFacetResults();
+ assertNotNull(facetResults);
+ assertEquals("incorrect label returned for RangeAccumulator", new CategoryPath("f"), facetResults.get(0).getFacetResultNode().label);
+
+ IOUtils.close(indexReader, taxoReader);
+
+ IOUtils.close(indexDir, taxoDir);
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java Sun Aug 11 12:19:13 2013
@@ -11,6 +11,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.search.FacetRequest.ResultMode;
@@ -150,8 +151,8 @@ public class TestTopKInEachNodeResultHan
FacetSearchParams facetSearchParams = new FacetSearchParams(iParams, facetRequests);
FacetArrays facetArrays = new FacetArrays(PartitionsUtils.partitionSize(facetSearchParams.indexingParams, tr));
- StandardFacetsAccumulator sfa = new StandardFacetsAccumulator(facetSearchParams, is.getIndexReader(), tr, facetArrays);
- sfa.setComplementThreshold(StandardFacetsAccumulator.DISABLE_COMPLEMENT);
+ OldFacetsAccumulator sfa = new OldFacetsAccumulator(facetSearchParams, is.getIndexReader(), tr, facetArrays);
+ sfa.setComplementThreshold(OldFacetsAccumulator.DISABLE_COMPLEMENT);
FacetsCollector fc = FacetsCollector.create(sfa);
is.search(q, fc);
@@ -183,7 +184,7 @@ public class TestTopKInEachNodeResultHan
}
// now rearrange
double [] expectedValues00 = { 6.0, 1.0, 5.0, 3.0, 2.0 };
- fr = sfa.createFacetResultsHandler(cfra23).rearrangeFacetResult(fr);
+ fr = sfa.createFacetResultsHandler(cfra23, sfa.createOrdinalValueResolver(cfra23)).rearrangeFacetResult(fr);
i = 0;
for (FacetResultNode node : parentRes.subResults) {
assertEquals(expectedValues00[i++], node.value, Double.MIN_VALUE);
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java Sun Aug 11 12:19:13 2013
@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
+import org.apache.lucene.facet.old.OldFacetsAccumulator;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -33,8 +34,8 @@ public class TestTopKResultsHandlerRando
throws IOException {
Query q = new MatchAllDocsQuery();
FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, fip);
- StandardFacetsAccumulator sfa = new StandardFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
- sfa.setComplementThreshold(doComplement ? StandardFacetsAccumulator.FORCE_COMPLEMENT : StandardFacetsAccumulator.DISABLE_COMPLEMENT);
+ OldFacetsAccumulator sfa = new OldFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
+ sfa.setComplementThreshold(doComplement ? OldFacetsAccumulator.FORCE_COMPLEMENT : OldFacetsAccumulator.DISABLE_COMPLEMENT);
FacetsCollector fc = FacetsCollector.create(sfa);
searcher.search(q, fc);
List<FacetResult> facetResults = fc.getFacetResults();
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java Sun Aug 11 12:19:13 2013
@@ -33,6 +33,7 @@ import org.apache.lucene.facet.search.Fa
import org.apache.lucene.facet.search.FacetResult;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -111,7 +112,7 @@ public class TestSortedSetDocValuesFacet
//SortedSetDocValuesCollector c = new SortedSetDocValuesCollector(state);
//SortedSetDocValuesCollectorMergeBySeg c = new SortedSetDocValuesCollectorMergeBySeg(state);
- FacetsCollector c = FacetsCollector.create(new SortedSetDocValuesAccumulator(fsp, state));
+ FacetsCollector c = FacetsCollector.create(new SortedSetDocValuesAccumulator(state, fsp));
searcher.search(new MatchAllDocsQuery(), c);
@@ -145,4 +146,51 @@ public class TestSortedSetDocValuesFacet
searcher.getIndexReader().close();
dir.close();
}
+
+ // LUCENE-5090
+ public void testStaleState() throws Exception {
+ assumeTrue("Test requires SortedSetDV support", defaultCodecSupportsSortedSet());
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+
+ SortedSetDocValuesFacetFields dvFields = new SortedSetDocValuesFacetFields();
+
+ Document doc = new Document();
+ dvFields.addFields(doc, Collections.singletonList(new CategoryPath("a", "foo")));
+ writer.addDocument(doc);
+
+ IndexReader r = writer.getReader();
+ SortedSetDocValuesReaderState state = new SortedSetDocValuesReaderState(r);
+
+ doc = new Document();
+ dvFields.addFields(doc, Collections.singletonList(new CategoryPath("a", "bar")));
+ writer.addDocument(doc);
+
+ doc = new Document();
+ dvFields.addFields(doc, Collections.singletonList(new CategoryPath("a", "baz")));
+ writer.addDocument(doc);
+
+ IndexSearcher searcher = newSearcher(writer.getReader());
+
+ List<FacetRequest> requests = new ArrayList<FacetRequest>();
+ requests.add(new CountFacetRequest(new CategoryPath("a"), 10));
+
+ FacetSearchParams fsp = new FacetSearchParams(requests);
+
+ FacetsCollector c = FacetsCollector.create(new SortedSetDocValuesAccumulator(state, fsp));
+
+ searcher.search(new MatchAllDocsQuery(), c);
+
+ try {
+ c.getFacetResults();
+ fail("did not hit expected exception");
+ } catch (IllegalStateException ise) {
+ // expected
+ }
+
+ r.close();
+ writer.close();
+ searcher.getIndexReader().close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestCategoryPath.java Sun Aug 11 12:19:13 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.taxonomy
import java.util.Arrays;
import org.apache.lucene.facet.FacetTestCase;
+import org.apache.lucene.util._TestUtil;
import org.junit.Test;
/*
@@ -274,5 +275,32 @@ public class TestCategoryPath extends Fa
// expected
}
}
+
+ @Test
+ public void testLongPath() throws Exception {
+ String bigComp = null;
+ while (true) {
+ int len = CategoryPath.MAX_CATEGORY_PATH_LENGTH;
+ bigComp = _TestUtil.randomSimpleString(random(), len, len);
+ if (bigComp.indexOf('\u001f') != -1) {
+ continue;
+ }
+ break;
+ }
+
+ try {
+ assertNotNull(new CategoryPath("dim", bigComp));
+ fail("long paths should not be allowed; len=" + bigComp.length());
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ try {
+ assertNotNull(new CategoryPath("dim\u001f" + bigComp, '\u001f'));
+ fail("long paths should not be allowed; len=" + bigComp.length());
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java Sun Aug 11 12:19:13 2013
@@ -1,13 +1,19 @@
package org.apache.lucene.facet.taxonomy.directory;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.params.FacetIndexingParams;
+import org.apache.lucene.facet.search.DrillDownQuery;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.MemoryOrdinalMap;
@@ -20,8 +26,11 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util._TestUtil;
import org.junit.Test;
/*
@@ -412,5 +421,52 @@ public class TestDirectoryTaxonomyWriter
taxoWriter.close();
dir.close();
}
-
+
+ @Test
+ public void testHugeLabel() throws Exception {
+ Directory indexDir = newDirectory(), taxoDir = newDirectory();
+ IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, new Cl2oTaxonomyWriterCache(2, 1f, 1));
+ FacetFields facetFields = new FacetFields(taxoWriter);
+
+ // Add one huge label:
+ String bigs = null;
+ int ordinal = -1;
+ CategoryPath cp = null;
+ while (true) {
+ int len = CategoryPath.MAX_CATEGORY_PATH_LENGTH - 4; // for the dimension and separator
+ bigs = _TestUtil.randomSimpleString(random(), len, len);
+ cp = new CategoryPath("dim", bigs);
+ ordinal = taxoWriter.addCategory(cp);
+ Document doc = new Document();
+ facetFields.addFields(doc, Collections.singletonList(cp));
+ indexWriter.addDocument(doc);
+ break;
+ }
+
+ // Add tiny ones to cause a re-hash
+ for (int i = 0; i < 3; i++) {
+ String s = _TestUtil.randomSimpleString(random(), 1, 10);
+ taxoWriter.addCategory(new CategoryPath("dim", s));
+ Document doc = new Document();
+ facetFields.addFields(doc, Collections.singletonList(new CategoryPath("dim", s)));
+ indexWriter.addDocument(doc);
+ }
+
+ // when too large components were allowed to be added, this resulted in a new added category
+ assertEquals(ordinal, taxoWriter.addCategory(cp));
+
+ IOUtils.close(indexWriter, taxoWriter);
+
+ DirectoryReader indexReader = DirectoryReader.open(indexDir);
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
+ IndexSearcher searcher = new IndexSearcher(indexReader);
+ DrillDownQuery ddq = new DrillDownQuery(FacetIndexingParams.DEFAULT);
+ ddq.add(cp);
+ assertEquals(1, searcher.search(ddq, 10).totalHits);
+
+ IOUtils.close(indexReader, taxoReader);
+
+ IOUtils.close(indexDir, taxoDir);
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java (original)
+++ lucene/dev/branches/lucene4956/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/cl2o/TestCharBlockArray.java Sun Aug 11 12:19:13 2013
@@ -11,6 +11,7 @@ import java.nio.charset.CodingErrorActio
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util._TestUtil;
import org.junit.Test;
/*
@@ -83,7 +84,8 @@ public class TestCharBlockArray extends
assertEqualsInternal("GrowingCharArray<->StringBuilder mismatch.", builder, array);
- File f = new File("GrowingCharArrayTest.tmp");
+ File tempDir = _TestUtil.getTempDir("growingchararray");
+ File f = new File(tempDir, "GrowingCharArrayTest.tmp");
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(f));
array.flush(out);
out.flush();
Modified: lucene/dev/branches/lucene4956/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java (original)
+++ lucene/dev/branches/lucene4956/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java Sun Aug 11 12:19:13 2013
@@ -303,7 +303,7 @@ public abstract class TermGroupFacetColl
int facetOrd;
if (groupedFacetHit.facetValue != null) {
- if (facetOrdTermsEnum == null || !facetOrdTermsEnum.seekExact(groupedFacetHit.facetValue, true)) {
+ if (facetOrdTermsEnum == null || !facetOrdTermsEnum.seekExact(groupedFacetHit.facetValue)) {
continue;
}
facetOrd = (int) facetOrdTermsEnum.ord();
@@ -319,7 +319,7 @@ public abstract class TermGroupFacetColl
if (facetPrefix != null) {
TermsEnum.SeekStatus seekStatus;
if (facetOrdTermsEnum != null) {
- seekStatus = facetOrdTermsEnum.seekCeil(facetPrefix, true);
+ seekStatus = facetOrdTermsEnum.seekCeil(facetPrefix);
} else {
seekStatus = TermsEnum.SeekStatus.END;
}
@@ -334,7 +334,7 @@ public abstract class TermGroupFacetColl
BytesRef facetEndPrefix = BytesRef.deepCopyOf(facetPrefix);
facetEndPrefix.append(UnicodeUtil.BIG_TERM);
- seekStatus = facetOrdTermsEnum.seekCeil(facetEndPrefix, true);
+ seekStatus = facetOrdTermsEnum.seekCeil(facetEndPrefix);
if (seekStatus != TermsEnum.SeekStatus.END) {
endFacetOrd = (int) facetOrdTermsEnum.ord();
} else {
Modified: lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java Sun Aug 11 12:19:13 2013
@@ -72,74 +72,78 @@ public class AllGroupHeadsCollectorTest
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- boolean canUseIDV = true;
DocValuesType valueType = vts[random().nextInt(vts.length)];
// 0
Document doc = new Document();
- addGroupField(doc, groupField, "author1", canUseIDV, valueType);
- doc.add(newTextField("content", "random text", Field.Store.YES));
- doc.add(newStringField("id", "1", Field.Store.YES));
+ addGroupField(doc, groupField, "author1", valueType);
+ doc.add(newTextField("content", "random text", Field.Store.NO));
+ doc.add(new IntField("id_1", 1, Field.Store.NO));
+ doc.add(newStringField("id_2", "1", Field.Store.NO));
w.addDocument(doc);
// 1
doc = new Document();
- addGroupField(doc, groupField, "author1", canUseIDV, valueType);
- doc.add(newTextField("content", "some more random text blob", Field.Store.YES));
- doc.add(newStringField("id", "2", Field.Store.YES));
+ addGroupField(doc, groupField, "author1", valueType);
+ doc.add(newTextField("content", "some more random text blob", Field.Store.NO));
+ doc.add(new IntField("id_1", 2, Field.Store.NO));
+ doc.add(newStringField("id_2", "2", Field.Store.NO));
w.addDocument(doc);
// 2
doc = new Document();
- addGroupField(doc, groupField, "author1", canUseIDV, valueType);
- doc.add(newTextField("content", "some more random textual data", Field.Store.YES));
- doc.add(newStringField("id", "3", Field.Store.YES));
+ addGroupField(doc, groupField, "author1", valueType);
+ doc.add(newTextField("content", "some more random textual data", Field.Store.NO));
+ doc.add(new IntField("id_1", 3, Field.Store.NO));
+ doc.add(newStringField("id_2", "3", Field.Store.NO));
w.addDocument(doc);
w.commit(); // To ensure a second segment
// 3
doc = new Document();
- addGroupField(doc, groupField, "author2", canUseIDV, valueType);
- doc.add(newTextField("content", "some random text", Field.Store.YES));
- doc.add(newStringField("id", "4", Field.Store.YES));
+ addGroupField(doc, groupField, "author2", valueType);
+ doc.add(newTextField("content", "some random text", Field.Store.NO));
+ doc.add(new IntField("id_1", 4, Field.Store.NO));
+ doc.add(newStringField("id_2", "4", Field.Store.NO));
w.addDocument(doc);
// 4
doc = new Document();
- addGroupField(doc, groupField, "author3", canUseIDV, valueType);
- doc.add(newTextField("content", "some more random text", Field.Store.YES));
- doc.add(newStringField("id", "5", Field.Store.YES));
+ addGroupField(doc, groupField, "author3", valueType);
+ doc.add(newTextField("content", "some more random text", Field.Store.NO));
+ doc.add(new IntField("id_1", 5, Field.Store.NO));
+ doc.add(newStringField("id_2", "5", Field.Store.NO));
w.addDocument(doc);
// 5
doc = new Document();
- addGroupField(doc, groupField, "author3", canUseIDV, valueType);
- doc.add(newTextField("content", "random blob", Field.Store.YES));
- doc.add(newStringField("id", "6", Field.Store.YES));
+ addGroupField(doc, groupField, "author3", valueType);
+ doc.add(newTextField("content", "random blob", Field.Store.NO));
+ doc.add(new IntField("id_1", 6, Field.Store.NO));
+ doc.add(newStringField("id_2", "6", Field.Store.NO));
w.addDocument(doc);
// 6 -- no author field
doc = new Document();
- doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.YES));
- doc.add(newStringField("id", "6", Field.Store.YES));
+ doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
+ doc.add(new IntField("id_1", 6, Field.Store.NO));
+ doc.add(newStringField("id_2", "6", Field.Store.NO));
w.addDocument(doc);
// 7 -- no author field
doc = new Document();
- doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.YES));
- doc.add(newStringField("id", "7", Field.Store.YES));
+ doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
+ doc.add(new IntField("id_1", 7, Field.Store.NO));
+ doc.add(newStringField("id_2", "7", Field.Store.NO));
w.addDocument(doc);
IndexReader reader = w.getReader();
IndexSearcher indexSearcher = newSearcher(reader);
- if (SlowCompositeReaderWrapper.class.isAssignableFrom(reader.getClass())) {
- canUseIDV = false;
- }
w.close();
int maxDoc = reader.maxDoc();
- Sort sortWithinGroup = new Sort(new SortField("id", SortField.Type.INT, true));
+ Sort sortWithinGroup = new Sort(new SortField("id_1", SortField.Type.INT, true));
AbstractAllGroupHeadsCollector<?> allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);
assertTrue(arrayContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads()));
@@ -156,13 +160,13 @@ public class AllGroupHeadsCollectorTest
assertTrue(openBitSetContains(new int[]{1, 5}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));
// STRING sort type triggers different implementation
- Sort sortWithinGroup2 = new Sort(new SortField("id", SortField.Type.STRING, true));
+ Sort sortWithinGroup2 = new Sort(new SortField("id_2", SortField.Type.STRING, true));
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup2);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);
assertTrue(arrayContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads()));
assertTrue(openBitSetContains(new int[]{2, 3, 5, 7}, allGroupHeadsCollector.retrieveGroupHeads(maxDoc), maxDoc));
- Sort sortWithinGroup3 = new Sort(new SortField("id", SortField.Type.STRING, false));
+ Sort sortWithinGroup3 = new Sort(new SortField("id_2", SortField.Type.STRING, false));
allGroupHeadsCollector = createRandomCollector(groupField, sortWithinGroup3);
indexSearcher.search(new TermQuery(new Term("content", "random")), allGroupHeadsCollector);
// 7 b/c higher doc id wins, even if order of field is in not in reverse.
@@ -402,6 +406,7 @@ public class AllGroupHeadsCollectorTest
for (int a : actual) {
if (e == a) {
found = true;
+ break;
}
}
@@ -539,11 +544,10 @@ public class AllGroupHeadsCollectorTest
return collector;
}
- private void addGroupField(Document doc, String groupField, String value, boolean canUseIDV, DocValuesType valueType) {
- doc.add(new TextField(groupField, value, Field.Store.YES));
- if (canUseIDV) {
- Field valuesField = null;
- switch(valueType) {
+ private void addGroupField(Document doc, String groupField, String value, DocValuesType valueType) {
+ doc.add(new TextField(groupField, value, Field.Store.NO));
+ Field valuesField = null;
+ switch(valueType) {
case BINARY:
valuesField = new BinaryDocValuesField(groupField + "_dv", new BytesRef(value));
break;
@@ -552,9 +556,8 @@ public class AllGroupHeadsCollectorTest
break;
default:
fail("unhandled type");
- }
- doc.add(valuesField);
}
+ doc.add(valuesField);
}
private static class GroupDoc {
Modified: lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/lucene4956/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Sun Aug 11 12:19:13 2013
@@ -827,12 +827,14 @@ public class TestGrouping extends Lucene
for(SortField sf : docSort.getSort()) {
if (sf.getType() == SortField.Type.SCORE) {
getScores = true;
+ break;
}
}
for(SortField sf : groupSort.getSort()) {
if (sf.getType() == SortField.Type.SCORE) {
getScores = true;
+ break;
}
}
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Sun Aug 11 12:19:13 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.search.highlig
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Iterator;
@@ -25,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.PriorityQueue;
/**
@@ -78,7 +76,7 @@ public class Highlighter
public final String getBestFragment(Analyzer analyzer, String fieldName,String text)
throws IOException, InvalidTokenOffsetsException
{
- TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(fieldName, text);
return getBestFragment(tokenStream, text);
}
@@ -130,7 +128,7 @@ public class Highlighter
int maxNumFragments)
throws IOException, InvalidTokenOffsetsException
{
- TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream(fieldName, text);
return getBestFragments(tokenStream, text, maxNumFragments);
}
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Sun Aug 11 12:19:13 2013
@@ -36,7 +36,6 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Comparator;
@@ -314,7 +313,7 @@ public class TokenSources {
public static TokenStream getTokenStream(String field, String contents,
Analyzer analyzer) {
try {
- return analyzer.tokenStream(field, new StringReader(contents));
+ return analyzer.tokenStream(field, contents);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Sun Aug 11 12:19:13 2013
@@ -282,7 +282,7 @@ public class WeightedSpanTermExtractor {
TreeSet<Term> extractedTerms = new TreeSet<Term>();
q.extractTerms(extractedTerms);
for (Term term : extractedTerms) {
- termContexts.put(term, TermContext.build(context, term, true));
+ termContexts.put(term, TermContext.build(context, term));
}
Bits acceptDocs = context.reader().getLiveDocs();
final Spans spans = q.getSpans(context, acceptDocs, termContexts);
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java Sun Aug 11 12:19:13 2013
@@ -369,7 +369,11 @@ public class PostingsHighlighter {
* identical to what was indexed. */
protected String[][] loadFieldValues(IndexSearcher searcher, String[] fields, int[] docids, int maxLength) throws IOException {
String contents[][] = new String[fields.length][docids.length];
- LimitedStoredFieldVisitor visitor = new LimitedStoredFieldVisitor(fields, maxLength);
+ char valueSeparators[] = new char[fields.length];
+ for (int i = 0; i < fields.length; i++) {
+ valueSeparators[i] = getMultiValuedSeparator(fields[i]);
+ }
+ LimitedStoredFieldVisitor visitor = new LimitedStoredFieldVisitor(fields, valueSeparators, maxLength);
for (int i = 0; i < docids.length; i++) {
searcher.doc(docids[i], visitor);
for (int j = 0; j < fields.length; j++) {
@@ -379,6 +383,16 @@ public class PostingsHighlighter {
}
return contents;
}
+
+ /**
+ * Returns the logical separator between values for multi-valued fields.
+ * The default value is a space character, which means passages can span across values,
+ * but a subclass can override, for example with {@code U+2029 PARAGRAPH SEPARATOR (PS)}
+ * if each value holds a discrete passage for highlighting.
+ */
+ protected char getMultiValuedSeparator(String field) {
+ return ' ';
+ }
private Map<Integer,String> highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List<AtomicReaderContext> leaves, int maxPassages) throws IOException {
Map<Integer,String> highlights = new HashMap<Integer,String>();
@@ -445,7 +459,7 @@ public class PostingsHighlighter {
continue;
} else if (de == null) {
postings[i] = EMPTY; // initially
- if (!termsEnum.seekExact(terms[i], true)) {
+ if (!termsEnum.seekExact(terms[i])) {
continue; // term not found
}
de = postings[i] = termsEnum.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_OFFSETS);
@@ -652,12 +666,15 @@ public class PostingsHighlighter {
private static class LimitedStoredFieldVisitor extends StoredFieldVisitor {
private final String fields[];
+ private final char valueSeparators[];
private final int maxLength;
private final StringBuilder builders[];
private int currentField = -1;
- public LimitedStoredFieldVisitor(String fields[], int maxLength) {
+ public LimitedStoredFieldVisitor(String fields[], char valueSeparators[], int maxLength) {
+ assert fields.length == valueSeparators.length;
this.fields = fields;
+ this.valueSeparators = valueSeparators;
this.maxLength = maxLength;
builders = new StringBuilder[fields.length];
for (int i = 0; i < builders.length; i++) {
@@ -669,8 +686,8 @@ public class PostingsHighlighter {
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
assert currentField >= 0;
StringBuilder builder = builders[currentField];
- if (builder.length() > 0) {
- builder.append(' '); // for the offset gap, TODO: make this configurable
+ if (builder.length() > 0 && builder.length() < maxLength) {
+ builder.append(valueSeparators[currentField]);
}
if (builder.length() + value.length() > maxLength) {
builder.append(value, 0, maxLength - builder.length());
Modified: lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java (original)
+++ lucene/dev/branches/lucene4956/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java Sun Aug 11 12:19:13 2013
@@ -60,50 +60,47 @@ public class FieldPhraseList {
public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit ){
final String field = fieldTermStack.getFieldName();
- LinkedList<TermInfo> phraseCandidate = new LinkedList<TermInfo>();
- QueryPhraseMap currMap = null;
- QueryPhraseMap nextMap = null;
- while( !fieldTermStack.isEmpty() && (phraseList.size() < phraseLimit) )
- {
- phraseCandidate.clear();
-
- TermInfo ti = fieldTermStack.pop();
- currMap = fieldQuery.getFieldTermMap( field, ti.getText() );
+ QueryPhraseMap qpm = fieldQuery.getRootMap(field);
+ if (qpm != null) {
+ LinkedList<TermInfo> phraseCandidate = new LinkedList<TermInfo>();
+ extractPhrases(fieldTermStack.termList, qpm, phraseCandidate, 0);
+ assert phraseCandidate.size() == 0;
+ }
+ }
- // if not found, discard top TermInfo from stack, then try next element
- if( currMap == null ) continue;
-
- // if found, search the longest phrase
- phraseCandidate.add( ti );
- while( true ){
- ti = fieldTermStack.pop();
- nextMap = null;
- if( ti != null )
- nextMap = currMap.getTermMap( ti.getText() );
- if( ti == null || nextMap == null ){
- if( ti != null )
- fieldTermStack.push( ti );
- if( currMap.isValidTermOrPhrase( phraseCandidate ) ){
- addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );
- }
- else{
- while( phraseCandidate.size() > 1 ){
- fieldTermStack.push( phraseCandidate.removeLast() );
- currMap = fieldQuery.searchPhrase( field, phraseCandidate );
- if( currMap != null ){
- addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );
- break;
- }
- }
- }
- break;
- }
- else{
- phraseCandidate.add( ti );
- currMap = nextMap;
+ void extractPhrases(LinkedList<TermInfo> terms, QueryPhraseMap currMap, LinkedList<TermInfo> phraseCandidate, int longest) {
+ if (terms.isEmpty()) {
+ if (longest > 0) {
+ addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate.subList(0, longest), currMap.getBoost(), currMap.getTermOrPhraseNumber() ) );
+ }
+ return;
+ }
+ ArrayList<TermInfo> samePositionTerms = new ArrayList<TermInfo>();
+ do {
+ samePositionTerms.add(terms.pop());
+ } while (!terms.isEmpty() && terms.get(0).getPosition() == samePositionTerms.get(0).getPosition());
+
+ // try all next terms at the same position
+ for (TermInfo nextTerm : samePositionTerms) {
+ QueryPhraseMap nextMap = currMap.getTermMap(nextTerm.getText());
+ if (nextMap != null) {
+ phraseCandidate.add(nextTerm);
+ int l = longest;
+ if(nextMap.isValidTermOrPhrase( phraseCandidate ) ){
+ l = phraseCandidate.size();
}
+ extractPhrases(terms, nextMap, phraseCandidate, l);
+ phraseCandidate.removeLast();
}
}
+
+ // ignore the next term
+ extractPhrases(terms, currMap, phraseCandidate, longest);
+
+ // add terms back
+ for (TermInfo nextTerm : samePositionTerms) {
+ terms.push(nextTerm);
+ }
}
public void addIfNoOverlap( WeightedPhraseInfo wpi ){
@@ -159,11 +156,11 @@ public class FieldPhraseList {
return termsInfos;
}
- public WeightedPhraseInfo( LinkedList<TermInfo> terms, float boost ){
+ public WeightedPhraseInfo( List<TermInfo> terms, float boost ){
this( terms, boost, 0 );
}
- public WeightedPhraseInfo( LinkedList<TermInfo> terms, float boost, int seqnum ){
+ public WeightedPhraseInfo( List<TermInfo> terms, float boost, int seqnum ){
this.boost = boost;
this.seqnum = seqnum;