You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by an...@apache.org on 2014/12/29 21:57:02 UTC
svn commit: r1648428 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/java/org/apache/solr/search/
solr/core/src/java/org/apache/solr/s...
Author: anshum
Date: Mon Dec 29 20:57:02 2014
New Revision: 1648428
URL: http://svn.apache.org/r1648428
Log:
SOLR-1632: Distributed IDF, finally. (merge from trunk)
Added:
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/
- copied from r1647253, lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/stats/
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/
- copied from r1647253, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrConfig.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
lucene/dev/branches/branch_5x/solr/solrj/ (props changed)
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Mon Dec 29 20:57:02 2014
@@ -210,6 +210,9 @@ New Features
* SOLR-6801: Load RequestHandler from blob store (Noble Paul)
+* SOLR-1632: Support Distributed IDF (Andrzej Bialecki, Mark Miller, Yonik Seeley,
+ Robert Muir, Markus Jelsma, Vitaliy Zhovtyuk, Anshum Gupta)
+
* SOLR-6729: createNodeSet.shuffle=(true|false) support for /admin/collections?action=CREATE.
(Christine Poerschke, Ramkumar Aiyengar via Mark Miller)
@@ -218,7 +221,6 @@ New Features
* SOLR-6851: Scripts to support installing and running Solr as a service on Linux
(Timothy Potter, Hossman, Steve Rowe)
-
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrConfig.java Mon Dec 29 20:57:02 2014
@@ -37,6 +37,7 @@ import org.apache.solr.search.CacheConfi
import org.apache.solr.search.FastLRUCache;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.ValueSourceParser;
+import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.servlet.SolrRequestParsers;
import org.apache.solr.spelling.QueryConverter;
import org.apache.solr.update.SolrIndexConfig;
@@ -263,7 +264,7 @@ public class SolrConfig extends Config i
jmxConfig = new JmxConfiguration(false, null, null, null);
}
maxWarmingSearchers = getInt("query/maxWarmingSearchers",Integer.MAX_VALUE);
- slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
+ slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
updateHandlerInfo = loadUpdatehandlerInfo();
@@ -324,6 +325,7 @@ public class SolrConfig extends Config i
.add(new SolrPluginInfo(IndexSchemaFactory.class, "schemaFactory", REQUIRE_CLASS))
.add(new SolrPluginInfo(RestManager.class, "restManager"))
.add(new SolrPluginInfo(InitParams.class, InitParams.TYPE, MULTI_OK))
+ .add(new SolrPluginInfo(StatsCache.class, "statsCache", REQUIRE_CLASS))
.build();
public static class SolrPluginInfo{
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrCore.java Mon Dec 29 20:57:02 2014
@@ -17,44 +17,6 @@
package org.apache.solr.core;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Writer;
-import java.lang.reflect.Constructor;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.NoSuchFileException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.IdentityHashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.StringTokenizer;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.commons.io.FileUtils;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
@@ -80,8 +42,6 @@ import org.apache.solr.core.DirectoryFac
import org.apache.solr.handler.ReplicationHandler;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.SnapPuller;
-import org.apache.solr.handler.SolrConfigHandler;
-import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.handler.admin.ShowFileRequestHandler;
import org.apache.solr.handler.component.DebugComponent;
import org.apache.solr.handler.component.ExpandComponent;
@@ -119,6 +79,8 @@ import org.apache.solr.search.QParserPlu
import org.apache.solr.search.SolrFieldCacheMBean;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.ValueSourceParser;
+import org.apache.solr.search.stats.LocalStatsCache;
+import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.update.DefaultSolrCoreState;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.SolrCoreState;
@@ -143,6 +105,43 @@ import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.lang.reflect.Constructor;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
/**
*
@@ -164,6 +163,8 @@ public final class SolrCore implements S
private boolean isReloaded = false;
+ private StatsCache statsCache;
+
private final SolrConfig solrConfig;
private final SolrResourceLoader resourceLoader;
private volatile IndexSchema schema;
@@ -818,6 +819,8 @@ public final class SolrCore implements S
// Handle things that should eventually go away
initDeprecatedSupport();
+ statsCache = initStatsCache();
+
// cause the executor to stall so firstSearcher events won't fire
// until after inform() has been called for all components.
// searchExecutor must be single-threaded for this to work
@@ -970,6 +973,27 @@ public final class SolrCore implements S
}
return factory.getCodec();
}
+
+ private StatsCache initStatsCache() {
+ final StatsCache cache;
+ PluginInfo pluginInfo = solrConfig.getPluginInfo(StatsCache.class.getName());
+ if (pluginInfo != null && pluginInfo.className != null && pluginInfo.className.length() > 0) {
+ cache = createInitInstance(pluginInfo, StatsCache.class, null,
+ LocalStatsCache.class.getName());
+ log.info("Using statsCache impl: " + cache.getClass().getName());
+ } else {
+ log.info("Using default statsCache cache: " + LocalStatsCache.class.getName());
+ cache = new LocalStatsCache();
+ }
+ return cache;
+ }
+
+ /**
+ * Get the StatsCache.
+ */
+ public StatsCache getStatsCache() {
+ return statsCache;
+ }
/**
* Load the request processors
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Mon Dec 29 20:57:02 2014
@@ -17,22 +17,8 @@
package org.apache.solr.handler.component;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
@@ -52,7 +38,13 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.*;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.apache.solr.common.params.GroupParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.MoreLikeThisParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@@ -71,11 +63,11 @@ import org.apache.solr.search.Grouping;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.RankQuery;
import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.search.SortSpec;
-import org.apache.solr.search.RankQuery;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.grouping.CommandHandler;
import org.apache.solr.search.grouping.GroupingSpecification;
@@ -96,8 +88,25 @@ import org.apache.solr.search.grouping.e
import org.apache.solr.search.grouping.endresulttransformer.GroupedEndResultTransformer;
import org.apache.solr.search.grouping.endresulttransformer.MainEndResultTransformer;
import org.apache.solr.search.grouping.endresulttransformer.SimpleEndResultTransformer;
+import org.apache.solr.search.stats.StatsCache;
import org.apache.solr.util.SolrPluginUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
/**
* TODO!
@@ -108,6 +117,7 @@ import java.util.Collections;
public class QueryComponent extends SearchComponent
{
public static final String COMPONENT_NAME = "query";
+ private static final Logger LOG = LoggerFactory.getLogger(QueryComponent.class);
@Override
public void prepare(ResponseBuilder rb) throws IOException
@@ -271,6 +281,8 @@ public class QueryComponent extends Sear
@Override
public void process(ResponseBuilder rb) throws IOException
{
+ LOG.debug("process: {}", rb.req.getParams());
+
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
@@ -279,6 +291,19 @@ public class QueryComponent extends Sear
}
SolrIndexSearcher searcher = req.getSearcher();
+ StatsCache statsCache = req.getCore().getStatsCache();
+
+ int purpose = params.getInt(ShardParams.SHARDS_PURPOSE, ShardRequest.PURPOSE_GET_TOP_IDS);
+ if ((purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
+ statsCache.returnLocalStats(rb, searcher);
+ return;
+ }
+ // check if we need to update the local copy of global dfs
+ if ((purpose & ShardRequest.PURPOSE_SET_TERM_STATS) != 0) {
+ // retrieve from request and update local cache
+ statsCache.receiveGlobalStats(req);
+ }
+
if (rb.getQueryCommand().getOffset() < 0) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
}
@@ -329,6 +354,9 @@ public class QueryComponent extends Sear
SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
cmd.setTimeAllowed(timeAllowed);
+
+ req.getContext().put(SolrIndexSearcher.STATS_SOURCE, statsCache.get(req));
+
SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
//
@@ -479,8 +507,8 @@ public class QueryComponent extends Sear
}
// normal search result
- searcher.search(result,cmd);
- rb.setResult( result );
+ searcher.search(result, cmd);
+ rb.setResult(result);
ResultContext ctx = new ResultContext();
ctx.docs = rb.getResults().docList;
@@ -637,7 +665,7 @@ public class QueryComponent extends Sear
if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY) {
nextStage = ResponseBuilder.STAGE_PARSE_QUERY;
} else if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
- createDistributedIdf(rb);
+ createDistributedStats(rb);
nextStage = ResponseBuilder.STAGE_TOP_GROUPS;
} else if (rb.stage < ResponseBuilder.STAGE_TOP_GROUPS) {
nextStage = ResponseBuilder.STAGE_TOP_GROUPS;
@@ -668,7 +696,7 @@ public class QueryComponent extends Sear
if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY)
return ResponseBuilder.STAGE_PARSE_QUERY;
if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) {
- createDistributedIdf(rb);
+ createDistributedStats(rb);
return ResponseBuilder.STAGE_EXECUTE_QUERY;
}
if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) return ResponseBuilder.STAGE_EXECUTE_QUERY;
@@ -713,6 +741,10 @@ public class QueryComponent extends Sear
mergeIds(rb, sreq);
}
+ if ((sreq.purpose & ShardRequest.PURPOSE_GET_TERM_STATS) != 0) {
+ updateStats(rb, sreq);
+ }
+
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
returnFields(rb, sreq);
}
@@ -786,8 +818,19 @@ public class QueryComponent extends Sear
}
}
- private void createDistributedIdf(ResponseBuilder rb) {
- // TODO
+ private void createDistributedStats(ResponseBuilder rb) {
+ StatsCache cache = rb.req.getCore().getStatsCache();
+ if ( (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)!=0 || rb.getSortSpec().includesScore()) {
+ ShardRequest sreq = cache.retrieveStatsRequest(rb);
+ if (sreq != null) {
+ rb.addRequest(this, sreq);
+ }
+ }
+ }
+
+ private void updateStats(ResponseBuilder rb, ShardRequest sreq) {
+ StatsCache cache = rb.req.getCore().getStatsCache();
+ cache.mergeToGlobalStats(rb.req, sreq.responses);
}
private void createMainQuery(ResponseBuilder rb) {
@@ -836,6 +879,12 @@ public class QueryComponent extends Sear
sreq.params.set(ResponseBuilder.FIELD_SORT_VALUES,"true");
+ // TODO: should this really sendGlobalDfs if just includeScore?
+ if ( (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES)!=0 || rb.getSortSpec().includesScore()) {
+ sreq.params.set(CommonParams.FL, rb.req.getSchema().getUniqueKeyField().getName() + ",score");
+ StatsCache statsCache = rb.req.getCore().getStatsCache();
+ statsCache.sendGlobalStats(rb, sreq);
+ }
boolean shardQueryIncludeScore = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0 || rb.getSortSpec().includesScore();
if (distribSinglePass) {
String[] fls = rb.req.getParams().getParams(CommonParams.FL);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java Mon Dec 29 20:57:02 2014
@@ -299,6 +299,7 @@ public class SearchHandler extends Reque
params.remove("indent");
params.remove(CommonParams.HEADER_ECHO_PARAMS);
params.set(ShardParams.IS_SHARD, true); // a sub (shard) request
+ params.set(ShardParams.SHARDS_PURPOSE, sreq.purpose);
params.set(ShardParams.SHARD_URL, shard); // so the shard knows what was asked
if (rb.requestInfo != null) {
// we could try and detect when this is needed, but it could be tricky
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java Mon Dec 29 20:57:02 2014
@@ -40,6 +40,8 @@ public class ShardRequest {
public final static int PURPOSE_GET_TOP_GROUPS =0x800;
public final static int PURPOSE_GET_MLT_RESULTS =0x1000;
public final static int PURPOSE_REFINE_PIVOT_FACETS =0x2000;
+ public final static int PURPOSE_SET_TERM_STATS =0x4000;
+ public final static int PURPOSE_GET_TERM_STATS = 0x8000;
public int purpose; // the purpose of this request
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon Dec 29 20:57:02 2014
@@ -58,11 +58,13 @@ import org.apache.lucene.index.MultiDocs
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSet;
@@ -80,6 +82,7 @@ import org.apache.lucene.search.SimpleCo
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TimeLimitingCollector;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
@@ -110,6 +113,7 @@ import org.apache.solr.request.UnInverte
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.stats.StatsSource;
import org.apache.solr.update.SolrIndexConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -124,6 +128,7 @@ import org.slf4j.LoggerFactory;
*/
public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrInfoMBean {
+ public static final String STATS_SOURCE = "org.apache.solr.stats_source";
// These should *only* be used for debugging or monitoring purposes
public static final AtomicLong numOpens = new AtomicLong();
public static final AtomicLong numCloses = new AtomicLong();
@@ -154,7 +159,7 @@ public class SolrIndexSearcher extends I
private final SolrCache<String,UnInvertedField> fieldValueCache;
private final LuceneQueryOptimizer optimizer;
-
+
// map of generic caches - not synchronized since it's read-only after the constructor.
private final HashMap<String, SolrCache> cacheMap;
private static final HashMap<String, SolrCache> noGenericCaches=new HashMap<>(0);
@@ -168,7 +173,7 @@ public class SolrIndexSearcher extends I
private final Collection<String> fieldNames;
private Collection<String> storedHighlightFieldNames;
private DirectoryFactory directoryFactory;
-
+
private final LeafReader leafReader;
// only for addIndexes etc (no fieldcache)
private final DirectoryReader rawReader;
@@ -321,7 +326,7 @@ public class SolrIndexSearcher extends I
// TODO: This option has been dead/noop since 3.1, should we re-enable it?
// optimizer = solrConfig.filtOptEnabled ? new LuceneQueryOptimizer(solrConfig.filtOptCacheSize,solrConfig.filtOptThreshold) : null;
optimizer = null;
-
+
fieldNames = new HashSet<>();
fieldInfos = leafReader.getFieldInfos();
for(FieldInfo fieldInfo : fieldInfos) {
@@ -331,6 +336,44 @@ public class SolrIndexSearcher extends I
// do this at the end since an exception in the constructor means we won't close
numOpens.incrementAndGet();
}
+
+ /*
+ * Override these two methods to provide a way to use global collection stats.
+ */
+ @Override
+ public TermStatistics termStatistics(Term term, TermContext context) throws IOException {
+ SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
+ if (reqInfo != null) {
+ StatsSource statsSrc = (StatsSource) reqInfo.getReq().getContext()
+ .get(STATS_SOURCE);
+ if (statsSrc != null) {
+ return statsSrc.termStatistics(this, term, context);
+ }
+ }
+ return localTermStatistics(term, context);
+ }
+
+ @Override
+ public CollectionStatistics collectionStatistics(String field)
+ throws IOException {
+ SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
+ if (reqInfo != null) {
+ StatsSource statsSrc = (StatsSource) reqInfo.getReq().getContext()
+ .get(STATS_SOURCE);
+ if (statsSrc != null) {
+ return statsSrc.collectionStatistics(this, field);
+ }
+ }
+ return localCollectionStatistics(field);
+ }
+
+ public TermStatistics localTermStatistics(Term term, TermContext context) throws IOException {
+ return super.termStatistics(term, context);
+ }
+
+ public CollectionStatistics localCollectionStatistics(String field) throws IOException {
+ return super.collectionStatistics(field);
+ }
public boolean isCachingEnabled() { return cachingEnabled; }
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java Mon Dec 29 20:57:02 2014
@@ -97,7 +97,7 @@ public class SolrPluginUtils {
static {
Map<Integer, String> map = new TreeMap<>();
map.put(ShardRequest.PURPOSE_PRIVATE, "PRIVATE");
- map.put(ShardRequest.PURPOSE_GET_TERM_DFS, "GET_TERM_DFS");
+ map.put(ShardRequest.PURPOSE_GET_TERM_STATS, "GET_TERM_STATS");
map.put(ShardRequest.PURPOSE_GET_TOP_IDS, "GET_TOP_IDS");
map.put(ShardRequest.PURPOSE_REFINE_TOP_IDS, "REFINE_TOP_IDS");
map.put(ShardRequest.PURPOSE_GET_FACETS, "GET_FACETS");
@@ -109,6 +109,7 @@ public class SolrPluginUtils {
map.put(ShardRequest.PURPOSE_GET_TERMS, "GET_TERMS");
map.put(ShardRequest.PURPOSE_GET_TOP_GROUPS, "GET_TOP_GROUPS");
map.put(ShardRequest.PURPOSE_GET_MLT_RESULTS, "GET_MLT_RESULTS");
+ map.put(ShardRequest.PURPOSE_SET_TERM_STATS, "SET_TERM_STATS");
purposes = Collections.unmodifiableMap(map);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Mon Dec 29 20:57:02 2014
@@ -52,6 +52,8 @@
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <statsCache class="${solr.statsCache:}"/>
+
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<updateHandler class="solr.DirectUpdateHandler2">
Modified: lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java?rev=1648428&r1=1648427&r2=1648428&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java (original)
+++ lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java Mon Dec 29 20:57:02 2014
@@ -49,6 +49,9 @@ public interface ShardParams {
/** Should things fail if there is an error? (true/false) */
@Deprecated
public static final String SHARD_KEYS = "shard.keys";
+
+ /** query purpose for shard requests */
+ public static final String SHARDS_PURPOSE = "shards.purpose";
public static final String _ROUTE_ = "_route_";