You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/02/26 01:03:41 UTC
svn commit: r1449973 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/java/org/apache/lucene/util/ lucene/facet/
lucene/facet/src/java/org/apache/lucene/facet/search/ lucene/fac...
Author: mikemccand
Date: Tue Feb 26 00:03:41 2013
New Revision: 1449973
URL: http://svn.apache.org/r1449973
Log:
LUCENE-4748: add DrillSideways utility class to facets module
Added:
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSideways.java
- copied unchanged from r1449972, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSideways.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysQuery.java
- copied unchanged from r1449972, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysQuery.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysScorer.java
- copied unchanged from r1449972, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/search/DrillSidewaysScorer.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java
- copied unchanged from r1449972, lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/search/TestDrillSideways.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDownQuery.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Feb 26 00:03:41 2013
@@ -61,6 +61,9 @@ Changes in backwards compatibility polic
should override FacetsAccumualtor and return the relevant aggregator,
for aggregating the association values. (Shai Erera)
+* LUCENE-4748: A FacetRequest on a non-existent field now returns an
+ empty FacetResult instead of skipping it. (Shai Erera, Mike McCandless)
+
Optimizations
* LUCENE-4687: BloomFilterPostingsFormat now lazily initializes delegate
@@ -164,6 +167,10 @@ New Features
* LUCENE-4780: Add MonotonicAppendingLongBuffer: an append-only buffer for
monotonically increasing values. (Adrien Grand)
+
+* LUCENE-4748: Added DrillSideways utility class for computing both
+ drill-down and drill-sideways counts for a DrillDownQuery. (Mike
+ McCandless)
API Changes
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java Tue Feb 26 00:03:41 2013
@@ -516,7 +516,7 @@ public class FilteredQuery extends Query
}
final Bits filterAcceptDocs = docIdSet.bits();
- // force if RA is requested
+ // force if RA is requested
final boolean useRandomAccess = (filterAcceptDocs != null && (useRandomAccess(filterAcceptDocs, firstFilterDoc)));
if (useRandomAccess) {
// if we are using random access, we return the inner scorer, just with other acceptDocs
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Tue Feb 26 00:03:41 2013
@@ -119,7 +119,7 @@ public final class FixedBitSet extends D
}
public void set(int index) {
- assert index >= 0 && index < numBits;
+ assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
int wordNum = index >> 6; // div 64
int bit = index & 0x3f; // mod 64
long bitmask = 1L << bit;
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDownQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDownQuery.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDownQuery.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/DrillDownQuery.java Tue Feb 26 00:03:41 2013
@@ -18,8 +18,8 @@ package org.apache.lucene.facet.search;
*/
import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.LinkedHashMap;
+import java.util.Map;
import org.apache.lucene.facet.params.CategoryListParams;
import org.apache.lucene.facet.params.FacetIndexingParams;
@@ -27,8 +27,11 @@ import org.apache.lucene.facet.taxonomy.
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -49,7 +52,7 @@ import org.apache.lucene.search.TermQuer
public final class DrillDownQuery extends Query {
/** Return a drill-down {@link Term} for a category. */
- public static final Term term(FacetIndexingParams iParams, CategoryPath path) {
+ public static Term term(FacetIndexingParams iParams, CategoryPath path) {
CategoryListParams clp = iParams.getCategoryListParams(path);
char[] buffer = new char[path.fullPathLength()];
iParams.drillDownTermText(path, buffer);
@@ -57,21 +60,37 @@ public final class DrillDownQuery extend
}
private final BooleanQuery query;
- private final Set<String> drillDownDims = new HashSet<String>();
-
+ private final Map<String,Integer> drillDownDims = new LinkedHashMap<String,Integer>();
private final FacetIndexingParams fip;
- /* Used by clone() */
- private DrillDownQuery(FacetIndexingParams fip, BooleanQuery query, Set<String> drillDownDims) {
+ /** Used by clone() */
+ DrillDownQuery(FacetIndexingParams fip, BooleanQuery query, Map<String,Integer> drillDownDims) {
this.fip = fip;
this.query = query.clone();
- this.drillDownDims.addAll(drillDownDims);
+ this.drillDownDims.putAll(drillDownDims);
+ }
+
+ /** Used by DrillSideways */
+ DrillDownQuery(Filter filter, DrillDownQuery other) {
+ query = new BooleanQuery(true); // disable coord
+
+ BooleanClause[] clauses = other.query.getClauses();
+ if (clauses.length == other.drillDownDims.size()) {
+ throw new IllegalArgumentException("cannot apply filter unless baseQuery isn't null; pass ConstantScoreQuery instead");
+ }
+ assert clauses.length == 1+other.drillDownDims.size(): clauses.length + " vs " + (1+other.drillDownDims.size());
+ drillDownDims.putAll(other.drillDownDims);
+ query.add(new FilteredQuery(clauses[0].getQuery(), filter), Occur.MUST);
+ for(int i=1;i<clauses.length;i++) {
+ query.add(clauses[i].getQuery(), Occur.MUST);
+ }
+ fip = other.fip;
}
/**
- * Creates a new {@link DrillDownQuery} without a base query, which means that
- * you intend to perfor a pure browsing query (equivalent to using
- * {@link MatchAllDocsQuery} as base.
+ * Creates a new {@link DrillDownQuery} without a base query,
+ * to perform a pure browsing query (equivalent to using
+ * {@link MatchAllDocsQuery} as base).
*/
public DrillDownQuery(FacetIndexingParams fip) {
this(fip, null);
@@ -97,14 +116,14 @@ public final class DrillDownQuery extend
*/
public void add(CategoryPath... paths) {
Query q;
+ if (paths[0].length == 0) {
+ throw new IllegalArgumentException("all CategoryPaths must have length > 0");
+ }
String dim = paths[0].components[0];
- if (drillDownDims.contains(dim)) {
+ if (drillDownDims.containsKey(dim)) {
throw new IllegalArgumentException("dimension '" + dim + "' was already added");
}
if (paths.length == 1) {
- if (paths[0].length == 0) {
- throw new IllegalArgumentException("all CategoryPaths must have length > 0");
- }
q = new TermQuery(term(fip, paths[0]));
} else {
BooleanQuery bq = new BooleanQuery(true); // disable coord
@@ -120,7 +139,7 @@ public final class DrillDownQuery extend
}
q = bq;
}
- drillDownDims.add(dim);
+ drillDownDims.put(dim, drillDownDims.size());
final ConstantScoreQuery drillDownQuery = new ConstantScoreQuery(q);
drillDownQuery.setBoost(0.0f);
@@ -162,5 +181,12 @@ public final class DrillDownQuery extend
public String toString(String field) {
return query.toString(field);
}
-
+
+ BooleanQuery getBooleanQuery() {
+ return query;
+ }
+
+ Map<String,Integer> getDims() {
+ return drillDownDims;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsAccumulator.java Tue Feb 26 00:03:41 2013
@@ -60,6 +60,26 @@ public class FacetsAccumulator {
public FacetsAccumulator(FacetSearchParams searchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
this(searchParams, indexReader, taxonomyReader, null);
}
+
+ /**
+ * Creates an appropriate {@link FacetsAccumulator},
+ * returning {@link FacetsAccumulator} when all requests
+ * are {@link CountFacetRequest} and only one partition is
+ * in use, otherwise {@link StandardFacetsAccumulator}.
+ */
+ public static FacetsAccumulator create(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
+ if (fsp.indexingParams.getPartitionSize() != Integer.MAX_VALUE) {
+ return new StandardFacetsAccumulator(fsp, indexReader, taxoReader);
+ }
+
+ for (FacetRequest fr : fsp.facetRequests) {
+ if (!(fr instanceof CountFacetRequest)) {
+ return new StandardFacetsAccumulator(fsp, indexReader, taxoReader);
+ }
+ }
+
+ return new FacetsAccumulator(fsp, indexReader, taxoReader);
+ }
/**
* Initializes the accumulator with the given parameters as well as
@@ -153,6 +173,12 @@ public class FacetsAccumulator {
for (FacetRequest fr : searchParams.facetRequests) {
int rootOrd = taxonomyReader.getOrdinal(fr.categoryPath);
if (rootOrd == TaxonomyReader.INVALID_ORDINAL) { // category does not exist
+ // Add empty FacetResult:
+ FacetResultNode root = new FacetResultNode();
+ root.ordinal = TaxonomyReader.INVALID_ORDINAL;
+ root.label = fr.categoryPath;
+ root.value = 0;
+ res.add(new FacetResult(fr, root, 0));
continue;
}
CategoryListParams clp = searchParams.indexingParams.getCategoryListParams(fr.categoryPath);
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java Tue Feb 26 00:03:41 2013
@@ -163,21 +163,11 @@ public abstract class FacetsCollector ex
}
/**
- * Creates a {@link FacetsCollector} with the default
- * {@link FacetsAccumulator}.
+ * Creates a {@link FacetsCollector} using the {@link
+ * FacetsAccumulator} from {@link FacetsAccumulator#create}.
*/
public static FacetsCollector create(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
- if (fsp.indexingParams.getPartitionSize() != Integer.MAX_VALUE) {
- return create(new StandardFacetsAccumulator(fsp, indexReader, taxoReader));
- }
-
- for (FacetRequest fr : fsp.facetRequests) {
- if (!(fr instanceof CountFacetRequest)) {
- return create(new StandardFacetsAccumulator(fsp, indexReader, taxoReader));
- }
- }
-
- return create(new FacetsAccumulator(fsp, indexReader, taxoReader));
+ return create(FacetsAccumulator.create(fsp, indexReader, taxoReader));
}
/**
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsAccumulator.java Tue Feb 26 00:03:41 2013
@@ -197,7 +197,13 @@ public class StandardFacetsAccumulator e
PartitionsFacetResultsHandler frHndlr = createFacetResultsHandler(fr);
IntermediateFacetResult tmpResult = fr2tmpRes.get(fr);
if (tmpResult == null) {
- continue; // do not add a null to the list.
+ // Add empty FacetResult:
+ FacetResultNode root = new FacetResultNode();
+ root.ordinal = TaxonomyReader.INVALID_ORDINAL;
+ root.label = fr.categoryPath;
+ root.value = 0;
+ res.add(new FacetResult(fr, root, 0));
+ continue;
}
FacetResult facetRes = frHndlr.renderFacetResult(tmpResult);
// final labeling if allowed (because labeling is a costly operation)
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java Tue Feb 26 00:03:41 2013
@@ -259,18 +259,30 @@ public class TestTopKInEachNodeResultHan
assertEquals(0, node.subResults.size());
}
- fr = facetResults.get(6); // a/b, depth=0, K=2
+ fr = facetResults.get(6); // Doctor, depth=0, K=2
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(0, fr.getNumValidDescendants()); // 0 descendants but rootnode
parentRes = fr.getFacetResultNode();
- assertEquals(8.0, parentRes.value, Double.MIN_VALUE);
+ assertEquals(0.0, parentRes.value, Double.MIN_VALUE);
assertEquals(0, parentRes.subResults.size());
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
// doctor, depth=1, K=2
- assertFalse("Shouldn't have found anything for a FacetRequest " +
- "of a facet that doesn't exist in the index.", hasDoctor);
- assertEquals("Shouldn't have found more than seven request.", 7, facetResults.size());
+ assertTrue("Should have found an empty FacetResult " +
+ "for a facet that doesn't exist in the index.", hasDoctor);
+ assertEquals("Shouldn't have found more than 8 request.", 8, facetResults.size());
+
+ fr = facetResults.get(7); // a/b, depth=0, K=2
+ assertEquals(0, fr.getNumValidDescendants());
+ parentRes = fr.getFacetResultNode();
+ assertEquals(8.0, parentRes.value, Double.MIN_VALUE);
+ assertEquals(0, parentRes.subResults.size());
+ i = 0;
+ for (FacetResultNode node : parentRes.subResults) {
+ assertEquals(expectedValues3[i++], node.value, Double.MIN_VALUE);
+ assertEquals(0, node.subResults.size());
+ }
+
ir.close();
tr.close();
iDir.close();
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java Tue Feb 26 00:03:41 2013
@@ -203,8 +203,8 @@ public class TestTopKResultsHandler exte
List<FacetResult> facetResults = fc.getFacetResults();
assertEquals("Shouldn't have found anything for a FacetRequest "
- + "of a facet that doesn't exist in the index.", 0, facetResults.size());
-
+ + "of a facet that doesn't exist in the index.", 1, facetResults.size());
+ assertEquals("Miau Hattulla", facetResults.get(0).getFacetResultNode().label.components[0]);
closeAll();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java?rev=1449973&r1=1449972&r2=1449973&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java Tue Feb 26 00:03:41 2013
@@ -98,6 +98,9 @@ public class AssertingIndexSearcher exte
@Override
public boolean scoresDocsOutOfOrder() {
+ // TODO: if this returns false, we should wrap
+ // Scorer with AssertingScorer that confirms docIDs
+ // are in order?
return w.scoresDocsOutOfOrder();
}
};