You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2015/06/14 00:34:46 UTC
svn commit: r1685342 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/search/
solr/core/src/java/org/apache/solr/search/facet/
solr/core/src/test/org/apache/solr/search/facet/
Author: yonik
Date: Sat Jun 13 22:34:45 2015
New Revision: 1685342
URL: http://svn.apache.org/r1685342
Log:
SOLR-7676: nested docs facet domain switching
Added:
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/BlockJoin.java
- copied unchanged from r1685340, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/facet/BlockJoin.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/QueryContext.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1685342&r1=1685341&r2=1685342&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Sat Jun 13 22:34:45 2015
@@ -56,6 +56,13 @@ New Features
* SOLR-7458: Expose HDFS Block Locality Metrics via JMX (Mike Drob via Mark Miller)
+* SOLR-7676: Faceting on nested objects / Block-join faceting with the new JSON Facet API.
+ Example: Assuming books with nested pages and an input domain of pages, the following
+ will switch the domain to books before faceting on the author field:
+ authors:{ type:terms, field:author, domain:{toParent:"type:book"} }
+ (yonik)
+
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/QueryContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/QueryContext.java?rev=1685342&r1=1685341&r2=1685342&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/QueryContext.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/QueryContext.java Sat Jun 13 22:34:45 2015
@@ -26,8 +26,10 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrRequestInfo;
-/**
- * Bridge between old style context and a real class
+/*
+ * Bridge between old style context and a real class.
+ * This is currently slightly more heavy weight than necessary because of the need to inherit from IdentityHashMap rather than
+ * instantiate it on demand (and the need to put "searcher" in the map)
* @lucene.experimental
*/
public class QueryContext extends IdentityHashMap implements Closeable {
@@ -45,7 +47,7 @@ public class QueryContext extends Identi
public QueryContext(IndexSearcher searcher) {
this.searcher = searcher instanceof SolrIndexSearcher ? (SolrIndexSearcher)searcher : null;
indexSearcher = searcher;
- this.put("searcher", searcher); // see ValueSource.newContext()
+ this.put("searcher", searcher); // see ValueSource.newContext() // TODO: move check to "get"?
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1685342&r1=1685341&r2=1685342&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sat Jun 13 22:34:45 2015
@@ -872,6 +872,25 @@ public class SolrIndexSearcher extends I
getDocSet(query);
}
+ public BitDocSet getDocSetBits(Query q) throws IOException {
+ DocSet answer = getDocSet(q);
+ if (answer instanceof BitDocSet) {
+ return (BitDocSet)answer;
+ }
+
+ FixedBitSet bs = new FixedBitSet(maxDoc());
+ DocIterator iter = answer.iterator();
+ while (iter.hasNext()) {
+ bs.set(iter.nextDoc());
+ }
+
+ BitDocSet answerBits = new BitDocSet(bs , answer.size());
+ if (filterCache != null) {
+ filterCache.put(q, answerBits);
+ }
+ return answerBits;
+ }
+
/**
* Returns the set of document ids matching a query.
* This method is cache-aware and attempts to retrieve the answer from the cache if possible.
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java?rev=1685342&r1=1685341&r2=1685342&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetQuery.java Sat Jun 13 22:34:45 2015
@@ -59,3 +59,4 @@ class FacetQueryProcessor extends FacetP
}
+
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java?rev=1685342&r1=1685341&r2=1685342&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java Sat Jun 13 22:34:45 2015
@@ -31,6 +31,7 @@ import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
@@ -41,6 +42,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.BitDocSet;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.FunctionQParser;
@@ -54,8 +56,17 @@ import org.apache.solr.search.SyntaxErro
public abstract class FacetRequest {
protected Map<String,AggValueSource> facetStats; // per-bucket statistics
protected Map<String,FacetRequest> subFacets; // list of facets
- protected List<String> excludeTags;
+ protected List<String> filters;
protected boolean processEmpty;
+ protected Domain domain;
+
+ // domain changes
+ public static class Domain {
+ public List<String> excludeTags;
+ public boolean toParent;
+ public boolean toChildren;
+ public String parents;
+ }
public FacetRequest() {
facetStats = new LinkedHashMap<>();
@@ -140,7 +151,42 @@ class FacetProcessor<FacetRequestT exten
}
protected void handleDomainChanges() throws IOException {
- if (freq.excludeTags == null || freq.excludeTags.size() == 0) {
+ if (freq.domain == null) return;
+ handleFilterExclusions();
+ handleBlockJoin();
+ }
+
+ private void handleBlockJoin() throws IOException {
+ if (!(freq.domain.toChildren || freq.domain.toParent)) return;
+
+ // TODO: avoid query parsing per-bucket somehow...
+ String parentStr = freq.domain.parents;
+ Query parentQuery;
+ try {
+ QParser parser = QParser.getParser(parentStr, null, fcontext.req);
+ parentQuery = parser.getQuery();
+ } catch (SyntaxError err) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing block join parent specification: " + parentStr);
+ }
+
+ BitDocSet parents = fcontext.searcher.getDocSetBits(parentQuery);
+ DocSet input = fcontext.base;
+ DocSet result;
+
+ if (freq.domain.toChildren) {
+ DocSet filt = fcontext.searcher.getDocSetBits( new MatchAllDocsQuery() );
+ result = BlockJoin.toChildren(input, parents, filt, fcontext.qcontext);
+ } else {
+ result = BlockJoin.toParents(input, parents, fcontext.qcontext);
+ }
+
+ fcontext.base = result;
+ }
+
+ private void handleFilterExclusions() throws IOException {
+ List<String> excludeTags = freq.domain.excludeTags;
+
+ if (excludeTags == null || excludeTags.size() == 0) {
return;
}
@@ -153,7 +199,7 @@ class FacetProcessor<FacetRequestT exten
}
IdentityHashMap<Query,Boolean> excludeSet = new IdentityHashMap<>();
- for (String excludeTag : freq.excludeTags) {
+ for (String excludeTag : excludeTags) {
Object olst = tagMap.get(excludeTag);
// tagMap has entries of List<String,List<QParser>>, but subject to change in the future
if (!(olst instanceof Collection)) continue;
@@ -487,7 +533,7 @@ abstract class FacetParser<FacetRequestT
} else if ("query".equals(type)) {
return parseQueryFacet(key, args);
} else if ("range".equals(type)) {
- return parseRangeFacet(key, args);
+ return parseRangeFacet(key, args);
}
return parseStat(key, type, args);
@@ -528,10 +574,41 @@ abstract class FacetParser<FacetRequestT
}
+ private FacetRequest.Domain getDomain() {
+ if (facet.domain == null) {
+ facet.domain = new FacetRequest.Domain();
+ }
+ return facet.domain;
+ }
+
protected void parseCommonParams(Object o) {
if (o instanceof Map) {
Map<String,Object> m = (Map<String,Object>)o;
- facet.excludeTags = getStringList(m, "excludeTags");
+ List<String> excludeTags = getStringList(m, "excludeTags");
+ if (excludeTags != null) {
+ getDomain().excludeTags = excludeTags;
+ }
+
+ Map<String,Object> domainMap = (Map<String,Object>) m.get("domain");
+ if (domainMap != null) {
+ excludeTags = getStringList(m, "excludeTags");
+ if (excludeTags != null) {
+ getDomain().excludeTags = excludeTags;
+ }
+
+ String blockParent = (String)domainMap.get("blockParent");
+ String blockChildren = (String)domainMap.get("blockChildren");
+
+ if (blockParent != null) {
+ getDomain().toParent = true;
+ getDomain().parents = blockParent;
+ } else if (blockChildren != null) {
+ getDomain().toChildren = true;
+ getDomain().parents = blockChildren;
+ }
+
+ }
+
}
}
@@ -696,6 +773,34 @@ class FacetQueryParser extends FacetPars
}
}
+/*** not a separate type of parser for now...
+class FacetBlockParentParser extends FacetParser<FacetBlockParent> {
+ public FacetBlockParentParser(FacetParser parent, String key) {
+ super(parent, key);
+ facet = new FacetBlockParent();
+ }
+
+ @Override
+ public FacetBlockParent parse(Object arg) throws SyntaxError {
+ parseCommonParams(arg);
+
+ if (arg instanceof String) {
+ // just the field name...
+ facet.parents = (String)arg;
+
+ } else if (arg instanceof Map) {
+ Map<String, Object> m = (Map<String, Object>) arg;
+ facet.parents = getString(m, "parents", null);
+
+ parseSubs( m.get("facet") );
+ }
+
+ return facet;
+ }
+}
+***/
+
+
class FacetFieldParser extends FacetParser<FacetField> {
public FacetFieldParser(FacetParser parent, String key) {
super(parent, key);
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java?rev=1685342&r1=1685341&r2=1685342&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java Sat Jun 13 22:34:45 2015
@@ -1072,6 +1072,82 @@ public class TestJsonFacets extends Solr
);
}
+
+
+ @Test
+ public void testBlockJoin() throws Exception {
+ doBlockJoin(Client.localClient());
+ }
+
+ public void doBlockJoin(Client client) throws Exception {
+ ModifiableSolrParams p = params("rows","0");
+
+ client.deleteByQuery("*:*", null);
+
+ SolrInputDocument parent;
+ parent = sdoc("id", "1", "type_s","book", "book_s","A", "v_t","q");
+ client.add(parent, null);
+
+ parent = sdoc("id", "2", "type_s","book", "book_s","B", "v_t","q w");
+ parent.addChildDocument( sdoc("id","2.1", "type_s","page", "page_s","a", "v_t","x y z") );
+ parent.addChildDocument( sdoc("id","2.2", "type_s","page", "page_s","b", "v_t","x y ") );
+ parent.addChildDocument( sdoc("id","2.3", "type_s","page", "page_s","c", "v_t"," y z" ) );
+ client.add(parent, null);
+
+ parent = sdoc("id", "3", "type_s","book", "book_s","C", "v_t","q w e");
+ parent.addChildDocument( sdoc("id","3.1", "type_s","page", "page_s","d", "v_t","x ") );
+ parent.addChildDocument( sdoc("id","3.2", "type_s","page", "page_s","e", "v_t"," y ") );
+ parent.addChildDocument( sdoc("id","3.3", "type_s","page", "page_s","f", "v_t"," z") );
+ client.add(parent, null);
+
+ parent = sdoc("id", "4", "type_s","book", "book_s","D", "v_t","e");
+ client.add(parent, null);
+
+ client.commit();
+
+ client.testJQ(params(p, "q", "*:*"
+ , "json.facet", "{ " +
+ "pages:{ type:query, domain:{blockChildren:'type_s:book'} , facet:{ x:{field:v_t} } }" +
+ ",pages2:{type:terms, field:v_t, domain:{blockChildren:'type_s:book'} }" +
+ ",books:{ type:query, domain:{blockParent:'type_s:book'} , facet:{ x:{field:v_t} } }" +
+ ",books2:{type:terms, field:v_t, domain:{blockParent:'type_s:book'} }" +
+ ",pageof3:{ type:query, q:'id:3', facet : { x : { type:terms, field:page_s, domain:{blockChildren:'type_s:book'}}} }" +
+ ",bookof22:{ type:query, q:'id:2.2', facet : { x : { type:terms, field:book_s, domain:{blockParent:'type_s:book'}}} }" +
+ ",missing_blockParent:{ type:query, domain:{blockParent:'type_s:does_not_exist'} }" +
+ ",missing_blockChildren:{ type:query, domain:{blockChildren:'type_s:does_not_exist'} }" +
+ "}"
+ )
+ , "facets=={ count:10" +
+ ", pages:{count:6 , x:{buckets:[ {val:y,count:4},{val:x,count:3},{val:z,count:3} ]} }" +
+ ", pages2:{ buckets:[ {val:y,count:4},{val:x,count:3},{val:z,count:3} ] }" +
+ ", books:{count:4 , x:{buckets:[ {val:q,count:3},{val:e,count:2},{val:w,count:2} ]} }" +
+ ", books2:{ buckets:[ {val:q,count:3},{val:e,count:2},{val:w,count:2} ] }" +
+ ", pageof3:{count:1 , x:{buckets:[ {val:d,count:1},{val:e,count:1},{val:f,count:1} ]} }" +
+ ", bookof22:{count:1 , x:{buckets:[ {val:B,count:1} ]} }" +
+ ", missing_blockParent:{count:0}" +
+ ", missing_blockChildren:{count:0}" +
+ "}"
+ );
+
+ // no matches in base query
+ client.testJQ(params("q", "no_match_s:NO_MATCHES"
+ , "json.facet", "{ processEmpty:true," +
+ "pages:{ type:query, domain:{blockChildren:'type_s:book'} }" +
+ ",books:{ type:query, domain:{blockParent:'type_s:book'} }" +
+ "}"
+ )
+ , "facets=={ count:0" +
+ ", pages:{count:0}" +
+ ", books:{count:0}" +
+ "}"
+ );
+
+
+ }
+
+
+
+
public void XtestPercentiles() {
AVLTreeDigest catA = new AVLTreeDigest(100);
catA.add(4);