You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/03/16 04:59:28 UTC
svn commit: r1577977 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/CHANGES.txt solr/core/
solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
Author: shalin
Date: Sun Mar 16 03:59:27 2014
New Revision: 1577977
URL: http://svn.apache.org/r1577977
Log:
SOLR-3177: Enable tagging and excluding filters in StatsComponent via the localParams syntax
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1577977&r1=1577976&r2=1577977&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sun Mar 16 03:59:27 2014
@@ -71,6 +71,9 @@ New Features
CollapsingQParserPlugin. (Joel Bernstein)
+* SOLR-3177: Enable tagging and excluding filters in StatsComponent via the
+ localParams syntax. (Mathias H., Nikolai Luthman, Vitaliy Zhovtyuk, shalin)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1577977&r1=1577976&r2=1577977&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Sun Mar 16 03:59:27 2014
@@ -19,18 +19,24 @@ package org.apache.solr.handler.componen
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.search.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.UnInvertedField;
import org.apache.solr.schema.FieldType;
@@ -38,7 +44,10 @@ import org.apache.solr.schema.IndexSchem
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryParsing;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SyntaxError;
/**
* Stats component calculates simple statistics on numeric field values
@@ -62,7 +71,8 @@ public class StatsComponent extends Sear
SolrParams params = rb.req.getParams();
SimpleStats s = new SimpleStats(rb.req,
rb.getResults().docSet,
- params );
+ params,
+ rb );
// TODO ???? add this directly to the response, or to the builder?
rb.rsp.add( "stats", s.getStatsCounts() );
@@ -194,23 +204,98 @@ class SimpleStats {
/** Searcher to use for all calculations */
protected SolrIndexSearcher searcher;
protected SolrQueryRequest req;
+ protected ResponseBuilder rb;
+
+ // per-stats values
+ SolrParams localParams;
+ String statsField;
+ DocSet base;
+ String key;
public SimpleStats(SolrQueryRequest req,
DocSet docs,
- SolrParams params) {
+ SolrParams params,
+ ResponseBuilder rb) {
this.req = req;
this.searcher = req.getSearcher();
this.docs = docs;
this.params = params;
+ this.rb = rb;
+ }
+
+ protected void parseParams(String param) throws SyntaxError, IOException {
+ localParams = QueryParsing.getLocalParams(param, req.getParams());
+ base = docs;
+ statsField = param;
+ key = param;
+
+ if (localParams == null) return;
+
+ statsField = localParams.get(CommonParams.VALUE);
+
+ // reset set the default key now that localParams have been removed
+ key = statsField;
+
+ // allow explicit set of the key
+ key = localParams.get(CommonParams.OUTPUT_KEY, key);
+
+
+ // figure out if we need a new base DocSet
+ String excludeStr = localParams.get(CommonParams.EXCLUDE);
+ if (excludeStr == null) return;
+
+ Map<?,?> tagMap = (Map<?,?>)req.getContext().get("tags");
+ if (tagMap != null && rb != null) {
+ List<String> excludeTagList = StrUtils.splitSmart(excludeStr,',');
+
+ IdentityHashMap<Query,Boolean> excludeSet = new IdentityHashMap<Query,Boolean>();
+ for (String excludeTag : excludeTagList) {
+ Object olst = tagMap.get(excludeTag);
+ // tagMap has entries of List<String,List<QParser>>, but subject to change in the future
+ if (!(olst instanceof Collection)) continue;
+ for (Object o : (Collection<?>)olst) {
+ if (!(o instanceof QParser)) continue;
+ QParser qp = (QParser)o;
+ excludeSet.put(qp.getQuery(), Boolean.TRUE);
+ }
+ }
+ if (excludeSet.size() == 0) return;
+
+ List<Query> qlist = new ArrayList<Query>();
+
+ // add the base query
+ if (!excludeSet.containsKey(rb.getQuery())) {
+ qlist.add(rb.getQuery());
+ }
+
+ // add the filters
+ if (rb.getFilters() != null) {
+ for (Query q : rb.getFilters()) {
+ if (!excludeSet.containsKey(q)) {
+ qlist.add(q);
+ }
+ }
+ }
+
+ // get the new base docset for this facet
+ this.base = searcher.getDocSet(qlist);
+ }
+
}
public NamedList<Object> getStatsCounts() throws IOException {
NamedList<Object> res = new SimpleOrderedMap<>();
- res.add("stats_fields", getStatsFields());
+
+ try {
+ res.add("stats_fields", getStatsFields());
+ } catch (SyntaxError e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, e);
+ }
+
return res;
}
- public NamedList<Object> getStatsFields() throws IOException {
+ public NamedList<Object> getStatsFields() throws IOException, SyntaxError {
NamedList<Object> res = new SimpleOrderedMap<>();
String[] statsFs = params.getParams(StatsParams.STATS_FIELD);
boolean isShard = params.getBool(ShardParams.IS_SHARD, false);
@@ -218,25 +303,28 @@ class SimpleStats {
final IndexSchema schema = searcher.getSchema();
for (String f : statsFs) {
boolean calcDistinct = params.getFieldBool(f, StatsParams.STATS_CALC_DISTINCT, false);
- String[] facets = params.getFieldParams(f, StatsParams.STATS_FACET);
+
+ parseParams(f);
+
+ String[] facets = params.getFieldParams(key, StatsParams.STATS_FACET);
if (facets == null) {
facets = new String[0]; // make sure it is something...
}
- SchemaField sf = schema.getField(f);
+ SchemaField sf = schema.getField(statsField);
FieldType ft = sf.getType();
NamedList<?> stv;
if (sf.multiValued() || ft.multiValuedFieldCache()) {
//use UnInvertedField for multivalued fields
- UnInvertedField uif = UnInvertedField.getUnInvertedField(f, searcher);
+ UnInvertedField uif = UnInvertedField.getUnInvertedField(statsField, searcher);
stv = uif.getStats(searcher, docs, calcDistinct, facets).getStatsValues();
} else {
- stv = getFieldCacheStats(f, calcDistinct, facets);
+ stv = getFieldCacheStats(statsField, calcDistinct, facets);
}
if (isShard == true || (Long) stv.get("count") > 0) {
- res.add(f, stv);
+ res.add(key, stv);
} else {
- res.add(f, null);
+ res.add(key, null);
}
}
}
@@ -263,7 +351,7 @@ class SimpleStats {
final Iterator<AtomicReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
AtomicReaderContext ctx = null;
- for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
+ for (DocIterator docsIt = base.iterator(); docsIt.hasNext(); ) {
final int doc = docsIt.nextDoc();
if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
// advance
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java?rev=1577977&r1=1577976&r2=1577977&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java Sun Mar 16 03:59:27 2014
@@ -395,4 +395,36 @@ public class StatsComponentTest extends
400);
}
+
+ //SOLR-3177
+ public void testStatsExcludeFilterQuery() throws Exception {
+ SolrCore core = h.getCore();
+ assertU(adoc("id", "1"));
+ assertU(adoc("id", "2"));
+ assertU(adoc("id", "3"));
+ assertU(adoc("id", "4"));
+ assertU(commit());
+
+ Map<String, String> args = new HashMap<String, String>();
+ args.put(CommonParams.Q, "*:*");
+ args.put(StatsParams.STATS, "true");
+ args.put(StatsParams.STATS_FIELD, "{!ex=id}id");
+ args.put("fq", "{!tag=id}id:[2 TO 3]");
+ SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+
+ assertQ("test exluding filter query", req
+ , "//lst[@name='id']/double[@name='min'][.='1.0']"
+ , "//lst[@name='id']/double[@name='max'][.='4.0']");
+
+ args = new HashMap<String, String>();
+ args.put(CommonParams.Q, "*:*");
+ args.put(StatsParams.STATS, "true");
+ args.put(StatsParams.STATS_FIELD, "{!key=id2}id");
+ args.put("fq", "{!tag=id}id:[2 TO 3]");
+ req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
+
+ assertQ("test rename field", req
+ , "//lst[@name='id2']/double[@name='min'][.='2.0']"
+ , "//lst[@name='id2']/double[@name='max'][.='3.0']");
+ }
}