You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2017/03/31 16:55:19 UTC
[1/2] lucene-solr:master: SOLR-7452: refinement of missing buckets
and partial facets through missing buckets
Repository: lucene-solr
Updated Branches:
refs/heads/master e80643e5a -> bdb0d588e
SOLR-7452: refinement of missing buckets and partial facets through missing buckets
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cc623403
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cc623403
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cc623403
Branch: refs/heads/master
Commit: cc623403bd82a176a057e1c9567f37d01f7391c4
Parents: e80643e
Author: yonik <yo...@apache.org>
Authored: Thu Mar 30 12:55:27 2017 -0400
Committer: yonik <yo...@apache.org>
Committed: Fri Mar 31 12:54:30 2017 -0400
----------------------------------------------------------------------
.../solr/search/facet/FacetFieldMerger.java | 25 +++++++++++++++++++
.../solr/search/facet/FacetFieldProcessor.java | 14 +++++++++++
.../search/facet/FacetRequestSortedMerger.java | 8 ++++++
.../search/facet/TestJsonFacetRefinement.java | 26 +++++++++++++++++++-
4 files changed, 72 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc623403/solr/core/src/java/org/apache/solr/search/facet/FacetFieldMerger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldMerger.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldMerger.java
index 63e8743..f8f6463 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldMerger.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldMerger.java
@@ -18,8 +18,11 @@
package org.apache.solr.search.facet;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.solr.common.util.SimpleOrderedMap;
@@ -167,7 +170,29 @@ public class FacetFieldMerger extends FacetRequestSortedMerger<FacetField> {
// basically , only do at the top-level facet?
}
+ @Override
+ Map<String, Object> getRefinementSpecial(Context mcontext, Map<String, Object> refinement, Collection<String> tagsWithPartial) {
+ if (!tagsWithPartial.isEmpty()) {
+ // Since special buckets missing and allBuckets themselves will always be included, we only need to worry about subfacets being partial.
+ if (freq.missing) {
+ refinement = getRefinementSpecial(mcontext, refinement, tagsWithPartial, missingBucket, "missing");
+ }
+ if (freq.allBuckets) {
+ refinement = getRefinementSpecial(mcontext, refinement, tagsWithPartial, allBuckets, "allBuckets");
+ }
+ }
+ return refinement;
+ }
+ private Map<String, Object> getRefinementSpecial(Context mcontext, Map<String, Object> refinement, Collection<String> tagsWithPartial, FacetBucket bucket, String label) {
+ // boolean prev = mcontext.setBucketWasMissing(true); // the special buckets should have the same "missing" status as this facet, so no need to set it again
+ Map<String, Object> bucketRefinement = bucket.getRefinement(mcontext, tagsWithPartial);
+ if (bucketRefinement != null) {
+ refinement = refinement == null ? new HashMap<>(2) : refinement;
+ refinement.put(label, bucketRefinement);
+ }
+ return refinement;
+ }
private static class FacetNumBucketsMerger extends FacetMerger {
long sumBuckets;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc623403/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
index a29e78d..d4daf08 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
@@ -36,6 +36,8 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocSet;
+import static org.apache.solr.search.facet.FacetContext.SKIP_FACET;
+
/**
* Facet processing based on field values. (not range nor by query)
* @see FacetField
@@ -528,6 +530,9 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
protected SimpleOrderedMap<Object> refineFacets() throws IOException {
+ boolean skipThisFacet = (fcontext.flags & SKIP_FACET) != 0;
+
+
List leaves = asList(fcontext.facetInfo.get("_l")); // We have not seen this bucket: do full faceting for this bucket, including all sub-facets
List<List> skip = asList(fcontext.facetInfo.get("_s")); // We have seen this bucket, so skip stats on it, and skip sub-facets except for the specified sub-facets that should calculate specified buckets.
List<List> partial = asList(fcontext.facetInfo.get("_p")); // We have not seen this bucket, do full faceting for this bucket, and most sub-facets... but some sub-facets are partial and should only visit specified buckets.
@@ -563,6 +568,15 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
bucketList.add( refineBucket(bucketVal, false, facetInfo ) );
}
+ if (freq.missing) {
+ Map<String,Object> bucketFacetInfo = (Map<String,Object>)fcontext.facetInfo.get("missing");
+
+ if (bucketFacetInfo != null || !skipThisFacet) {
+ SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
+ fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null, skipThisFacet, bucketFacetInfo);
+ res.add("missing", missingBucket);
+ }
+ }
// If there are just a couple of leaves, and if the domain is large, then
// going by term is likely the most efficient?
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc623403/solr/core/src/java/org/apache/solr/search/facet/FacetRequestSortedMerger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRequestSortedMerger.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRequestSortedMerger.java
index e05064c..9ffdea7 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRequestSortedMerger.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRequestSortedMerger.java
@@ -240,6 +240,14 @@ abstract class FacetRequestSortedMerger<FacetRequestT extends FacetRequestSorted
if (skipBuckets != null) refinement.put("_s", skipBuckets);
}
+ refinement = getRefinementSpecial(mcontext, refinement, tagsWithPartial);
+
+ return refinement;
+ }
+
+ // utility method for subclasses to override to finish calculating faceting (special buckets in field facets)... this feels hacky and we
+ // should find a better way.
+ Map<String,Object> getRefinementSpecial(Context mcontext, Map<String,Object> refinement, Collection<String> tagsWithPartial) {
return refinement;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc623403/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
index b4b0220..52b8be4 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
@@ -219,6 +219,17 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
"}"
);
+ // test partial _p under a missing bucket
+ doTestRefine("{top:{type:terms, field:Afield, refine:true, limit:1, missing:true, facet:{x : {type:terms, field:X, limit:1, refine:true} } } }",
+ "{top: {buckets:[], missing:{count:12, x:{buckets:[{val:x2, count:4},{val:x3, count:2}]} } } }",
+ "{top: {buckets:[], missing:{count:10, x:{buckets:[{val:x1, count:5},{val:x4, count:3}]} } } }",
+ "=={top: {" +
+ "missing:{x:{_l:[x1]}}" +
+ " } " +
+ "}"
+ , null
+ );
+
}
@@ -266,6 +277,17 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
);
****/
+ // test refining under the special "missing" bucket of a field facet
+ client.testJQ(params(p, "q", "*:*",
+ "json.facet", "{" +
+ "f:{type:terms, field:missing_s, limit:1, overrequest:0, missing:true, refine:true, facet:{ cat:{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:true } }}" +
+ "}"
+ )
+ , "facets=={ count:8" +
+ ", f:{ buckets:[], missing:{count:8, cat:{buckets:[{val:A,count:4}]} } }" + // just like the previous response, just nested under a field facet
+ "}"
+ );
+
client.testJQ(params(p, "q", "*:*",
"json.facet", "{" +
@@ -317,7 +339,7 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
"}"
);
- // test missing buckets (field facet within field facet)
+ // test partial buckets (field facet within field facet)
client.testJQ(params(p, "q", "*:*",
"json.facet", "{" +
"ab:{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:true, facet:{ xy:{type:terms, field:${xy_s}, limit:1, overrequest:0, refine:true } }}" +
@@ -345,6 +367,8 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
"}"
);
+
+
}
[2/2] lucene-solr:master: SOLR-7452: add more tests for refinement of
missing buckets
Posted by yo...@apache.org.
SOLR-7452: add more tests for refinement of missing buckets
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bdb0d588
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bdb0d588
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bdb0d588
Branch: refs/heads/master
Commit: bdb0d588ee8129cf42df03f6185fbb3f4d8e0af4
Parents: cc62340
Author: yonik <yo...@apache.org>
Authored: Fri Mar 31 12:55:15 2017 -0400
Committer: yonik <yo...@apache.org>
Committed: Fri Mar 31 12:55:15 2017 -0400
----------------------------------------------------------------------
.../search/facet/TestJsonFacetRefinement.java | 62 +++++++++++++-------
1 file changed, 41 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bdb0d588/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
index 52b8be4..bcb5f09 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetRefinement.java
@@ -244,21 +244,22 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
client.deleteByQuery("*:*", null);
- ModifiableSolrParams p = params("cat_s", "cat_s", "xy_s", "xy_s", "num_d", "num_d");
+ ModifiableSolrParams p = params("cat_s", "cat_s", "xy_s", "xy_s", "num_d", "num_d", "qw_s", "qw_s");
String cat_s = p.get("cat_s");
String xy_s = p.get("xy_s");
+ String qw_s = p.get("qw_s");
String num_d = p.get("num_d");
- clients.get(0).add( sdoc("id", "01", "all_s","all", cat_s, "A", xy_s, "X" ,num_d, -1) ); // A wins count tie
- clients.get(0).add( sdoc("id", "02", "all_s","all", cat_s, "B", xy_s, "Y", num_d, 3) );
+ clients.get(0).add( sdoc("id", "01", "all_s","all", cat_s, "A", xy_s, "X" ,num_d, -1, qw_s, "Q") ); // A wins count tie
+ clients.get(0).add( sdoc("id", "02", "all_s","all", cat_s, "B", xy_s, "Y", num_d, 3 ) );
- clients.get(1).add( sdoc("id", "11", "all_s","all", cat_s, "B", xy_s, "X", num_d, -5) ); // B highest count
- clients.get(1).add( sdoc("id", "12", "all_s","all", cat_s, "B", xy_s, "Y", num_d, -11) );
- clients.get(1).add( sdoc("id", "13", "all_s","all", cat_s, "A", xy_s, "X", num_d, 7) );
+ clients.get(1).add( sdoc("id", "11", "all_s","all", cat_s, "B", xy_s, "X", num_d, -5 ) ); // B highest count
+ clients.get(1).add( sdoc("id", "12", "all_s","all", cat_s, "B", xy_s, "Y", num_d, -11, qw_s, "W") );
+ clients.get(1).add( sdoc("id", "13", "all_s","all", cat_s, "A", xy_s, "X", num_d, 7 ) );
- clients.get(2).add( sdoc("id", "21", "all_s","all", cat_s, "A", xy_s, "X", num_d, 17) ); // A highest count
- clients.get(2).add( sdoc("id", "22", "all_s","all", cat_s, "A", xy_s, "Y", num_d, -19) );
- clients.get(2).add( sdoc("id", "23", "all_s","all", cat_s, "B", xy_s, "X", num_d, 11) );
+ clients.get(2).add( sdoc("id", "21", "all_s","all", cat_s, "A", xy_s, "X", num_d, 17, qw_s, "W") ); // A highest count
+ clients.get(2).add( sdoc("id", "22", "all_s","all", cat_s, "A", xy_s, "Y", num_d, -19 ) );
+ clients.get(2).add( sdoc("id", "23", "all_s","all", cat_s, "B", xy_s, "X", num_d, 11 ) );
client.commit();
@@ -277,18 +278,6 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
);
****/
- // test refining under the special "missing" bucket of a field facet
- client.testJQ(params(p, "q", "*:*",
- "json.facet", "{" +
- "f:{type:terms, field:missing_s, limit:1, overrequest:0, missing:true, refine:true, facet:{ cat:{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:true } }}" +
- "}"
- )
- , "facets=={ count:8" +
- ", f:{ buckets:[], missing:{count:8, cat:{buckets:[{val:A,count:4}]} } }" + // just like the previous response, just nested under a field facet
- "}"
- );
-
-
client.testJQ(params(p, "q", "*:*",
"json.facet", "{" +
"cat0:{type:terms, field:${cat_s}, sort:'count desc', limit:1, overrequest:0, refine:false}" +
@@ -367,6 +356,37 @@ public class TestJsonFacetRefinement extends SolrTestCaseHS {
"}"
);
+ // test refining under the special "missing" bucket of a field facet
+ client.testJQ(params(p, "q", "*:*",
+ "json.facet", "{" +
+ "f:{type:terms, field:missing_s, limit:1, overrequest:0, missing:true, refine:true, facet:{ cat:{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:true } }}" +
+ "}"
+ )
+ , "facets=={ count:8" +
+ ", f:{ buckets:[], missing:{count:8, cat:{buckets:[{val:A,count:4}]} } }" + // just like the previous response, just nested under a field facet
+ "}"
+ );
+
+ // test filling in "missing" bucket for partially refined facets
+ client.testJQ(params(p, "q", "*:*",
+ "json.facet", "{" +
+ // test all values missing in sub-facet
+ " ab :{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:false, facet:{ zz:{type:terms, field:missing_s, limit:1, overrequest:0, refine:false, missing:true} }}" +
+ ",ab2:{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:true , facet:{ zz:{type:terms, field:missing_s, limit:1, overrequest:0, refine:true , missing:true} }}" +
+ // test some values missing in sub-facet (and test that this works with normal partial bucket refinement)
+ ", cd :{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:false, facet:{ qw:{type:terms, field:${qw_s}, limit:1, overrequest:0, refine:false, missing:true, facet:{qq:{query:'*:*'}} } }}" +
+ ", cd2:{type:terms, field:${cat_s}, limit:1, overrequest:0, refine:true , facet:{ qw:{type:terms, field:${qw_s}, limit:1, overrequest:0, refine:true , missing:true, facet:{qq:{query:'*:*'}} } }}" +
+
+ "}"
+ )
+ , "facets=={ count:8" +
+ ", ab:{ buckets:[ {val:A, count:3, zz:{buckets:[], missing:{count:3}}}] }" +
+ ",ab2:{ buckets:[ {val:A, count:4, zz:{buckets:[], missing:{count:4}}}] }" +
+ ", cd:{ buckets:[ {val:A, count:3, qw:{buckets:[{val:Q, count:1, qq:{count:1}}], missing:{count:1,qq:{count:1}}}}] }" +
+ ",cd2:{ buckets:[ {val:A, count:4, qw:{buckets:[{val:Q, count:1, qq:{count:1}}], missing:{count:2,qq:{count:2}}}}] }" +
+ "}"
+ );
+
}