You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2015/08/05 14:07:30 UTC
svn commit: r1694193 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/search/
solr/core/src/java/org/apache/solr/search/stats/
solr/core/src/test-files/solr/configsets/configset-2/conf/
solr/core/src/test/org/...
Author: varun
Date: Wed Aug 5 12:07:30 2015
New Revision: 1694193
URL: http://svn.apache.org/r1694193
Log:
SOLR-7756: ExactStatsCache and LRUStatsCache will throw an NPE when a term is not present on a shard (merged trunk r1694182)
Added:
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
- copied unchanged from r1694182, lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/StatsUtil.java
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1694193&r1=1694192&r2=1694193&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed Aug 5 12:07:30 2015
@@ -205,6 +205,9 @@ Bug Fixes
* SOLR-6357: Allow delete documents by doing a score join query. (Mikhail Khludnev, Timothy Potter)
+* SOLR-7756: ExactStatsCache and LRUStatsCache will throw an NPE when a term is not present on a shard.
+ (Varun Thacker, Anshum Gupta)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java?rev=1694193&r1=1694192&r2=1694193&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java Wed Aug 5 12:07:30 2015
@@ -53,9 +53,8 @@ import java.util.Set;
* query terms (and collection statistics for term fields).
*/
public class ExactStatsCache extends StatsCache {
- private static final Logger LOG = LoggerFactory
- .getLogger(ExactStatsCache.class);
-
+ private static final Logger LOG = LoggerFactory.getLogger(ExactStatsCache.class);
+
// experimenting with strategy that takes more RAM, but also doesn't share memory
// across threads
private static final String CURRENT_GLOBAL_COL_STATS = "org.apache.solr.stats.currentGlobalColStats";
@@ -68,7 +67,7 @@ public class ExactStatsCache extends Sta
Map<String,CollectionStats> currentGlobalColStats = (Map<String,CollectionStats>) req.getContext().get(CURRENT_GLOBAL_COL_STATS);
Map<String,TermStats> currentGlobalTermStats = (Map<String,TermStats>) req.getContext().get(CURRENT_GLOBAL_TERM_STATS);
if (currentGlobalColStats == null) {
- currentGlobalColStats = Collections.emptyMap();
+ currentGlobalColStats = Collections.emptyMap();
}
if (currentGlobalTermStats == null) {
currentGlobalTermStats = Collections.emptyMap();
@@ -76,10 +75,10 @@ public class ExactStatsCache extends Sta
LOG.debug("Returning StatsSource. Collection stats={}, Term stats size= {}", currentGlobalColStats, currentGlobalTermStats.size());
return new ExactStatsSource(currentGlobalTermStats, currentGlobalColStats);
}
-
+
@Override
public void init(PluginInfo info) {}
-
+
@Override
public ShardRequest retrieveStatsRequest(ResponseBuilder rb) {
ShardRequest sreq = new ShardRequest();
@@ -89,19 +88,17 @@ public class ExactStatsCache extends Sta
sreq.params.remove(ShardParams.SHARDS);
return sreq;
}
-
+
@Override
- public void mergeToGlobalStats(SolrQueryRequest req,
- List<ShardResponse> responses) {
+ public void mergeToGlobalStats(SolrQueryRequest req, List<ShardResponse> responses) {
for (ShardResponse r : responses) {
LOG.debug("Merging to global stats, shard={}, response={}", r.getShard(), r.getSolrResponse().getResponse());
String shard = r.getShard();
SolrResponse res = r.getSolrResponse();
NamedList<Object> nl = res.getResponse();
-
+
// TODO: nl == null if not all shards respond (no server hosting shard)
String termStatsString = (String) nl.get(TERM_STATS_KEY);
-
if (termStatsString != null) {
addToPerShardTermStats(req, shard, termStatsString);
}
@@ -109,23 +106,20 @@ public class ExactStatsCache extends Sta
if (terms != null) {
req.getContext().put(TERMS_KEY, terms);
}
+
String colStatsString = (String) nl.get(COL_STATS_KEY);
- if (colStatsString != null) {
- Map<String,CollectionStats> colStats = StatsUtil
- .colStatsMapFromString(colStatsString);
- if (colStats != null) {
- addToPerShardColStats(req, shard, colStats);
- }
+ Map<String,CollectionStats> colStats = StatsUtil.colStatsMapFromString(colStatsString);
+ if (colStats != null) {
+ addToPerShardColStats(req, shard, colStats);
}
}
if (LOG.isDebugEnabled()) printStats(req);
}
- protected void addToPerShardColStats(SolrQueryRequest req, String shard,
- Map<String,CollectionStats> colStats) {
+ protected void addToPerShardColStats(SolrQueryRequest req, String shard, Map<String,CollectionStats> colStats) {
Map<String,Map<String,CollectionStats>> perShardColStats = (Map<String,Map<String,CollectionStats>>) req.getContext().get(PER_SHARD_COL_STATS);
if (perShardColStats == null) {
- perShardColStats = new HashMap<String,Map<String,CollectionStats>>();
+ perShardColStats = new HashMap<>();
req.getContext().put(PER_SHARD_COL_STATS, perShardColStats);
}
perShardColStats.put(shard, colStats);
@@ -144,72 +138,72 @@ public class ExactStatsCache extends Sta
}
protected void addToPerShardTermStats(SolrQueryRequest req, String shard, String termStatsString) {
- Map<String,TermStats> termStats = StatsUtil
- .termStatsMapFromString(termStatsString);
+ Map<String,TermStats> termStats = StatsUtil.termStatsMapFromString(termStatsString);
if (termStats != null) {
Map<String,Map<String,TermStats>> perShardTermStats = (Map<String,Map<String,TermStats>>) req.getContext().get(PER_SHARD_TERM_STATS);
if (perShardTermStats == null) {
- perShardTermStats = new HashMap<String,Map<String,TermStats>>();
+ perShardTermStats = new HashMap<>();
req.getContext().put(PER_SHARD_TERM_STATS, perShardTermStats);
}
perShardTermStats.put(shard, termStats);
}
}
-
+
@Override
public void returnLocalStats(ResponseBuilder rb, SolrIndexSearcher searcher) {
Query q = rb.getQuery();
try {
- HashSet<Term> terms = new HashSet<Term>();
+ HashSet<Term> terms = new HashSet<>();
searcher.createNormalizedWeight(q, true).extractTerms(terms);
IndexReaderContext context = searcher.getTopReaderContext();
- HashMap<String,TermStats> statsMap = new HashMap<String,TermStats>();
- HashMap<String,CollectionStats> colMap = new HashMap<String,CollectionStats>();
+ HashMap<String,TermStats> statsMap = new HashMap<>();
+ HashMap<String,CollectionStats> colMap = new HashMap<>();
for (Term t : terms) {
TermContext termContext = TermContext.build(context, t);
-
+
TermStatistics tst = searcher.localTermStatistics(t, termContext);
if (tst.docFreq() == 0) { // skip terms that are not present here
continue;
}
-
+
statsMap.put(t.toString(), new TermStats(t.field(), tst));
rb.rsp.add(TERMS_KEY, t.toString());
if (!colMap.containsKey(t.field())) { // collection stats for this field
- colMap.put(
- t.field(),
- new CollectionStats(searcher.localCollectionStatistics(t.field())));
+ colMap.put(t.field(), new CollectionStats(searcher.localCollectionStatistics(t.field())));
}
}
-
- String termStatsString = StatsUtil.termStatsMapToString(statsMap);
- rb.rsp.add(TERM_STATS_KEY, termStatsString);
- String colStatsString = StatsUtil.colStatsMapToString(colMap);
- rb.rsp.add(COL_STATS_KEY, colStatsString);
- if (LOG.isDebugEnabled()) {
- LOG.debug("termStats=" + termStatsString + ", collectionStats="
- + colStatsString + ", terms=" + terms + ", numDocs="
- + searcher.maxDoc());
+ if (statsMap.size() != 0 && colMap.size() != 0) { //Don't add empty keys
+ String termStatsString = StatsUtil.termStatsMapToString(statsMap);
+ rb.rsp.add(TERM_STATS_KEY, termStatsString);
+
+ String colStatsString = StatsUtil.colStatsMapToString(colMap);
+ rb.rsp.add(COL_STATS_KEY, colStatsString);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("termStats=" + termStatsString + ", collectionStats="
+ + colStatsString + ", terms=" + terms + ", numDocs=" + searcher.maxDoc());
+ }
}
+
} catch (IOException e) {
LOG.error("Error collecting local stats, query='" + q.toString() + "'", e);
throw new SolrException(ErrorCode.SERVER_ERROR, "Error collecting local stats.", e);
}
}
-
+
@Override
public void sendGlobalStats(ResponseBuilder rb, ShardRequest outgoing) {
outgoing.purpose |= ShardRequest.PURPOSE_SET_TERM_STATS;
ModifiableSolrParams params = outgoing.params;
List<String> terms = (List<String>) rb.req.getContext().get(TERMS_KEY);
if (terms != null) {
- Set<String> fields = new HashSet<String>();
+ Set<String> fields = new HashSet<>();
for (String t : terms) {
String[] fv = t.split(":");
fields.add(fv[0]);
}
- Map<String,TermStats> globalTermStats = new HashMap<String,TermStats>();
- Map<String,CollectionStats> globalColStats = new HashMap<String,CollectionStats>();
+ Map<String,TermStats> globalTermStats = new HashMap<>();
+ Map<String,CollectionStats> globalColStats = new HashMap<>();
// aggregate collection stats, only for the field in terms
for (String shard : rb.shards) {
@@ -266,16 +260,15 @@ public class ExactStatsCache extends Sta
perShardTermStats = Collections.emptyMap();
}
Map<String,TermStats> cache = perShardTermStats.get(shard);
- return cache.get(t);
+ return (cache != null) ? cache.get(t) : null; //Term doesn't exist in shard
}
-
+
@Override
public void receiveGlobalStats(SolrQueryRequest req) {
String globalTermStats = req.getParams().get(TERM_STATS_KEY);
String globalColStats = req.getParams().get(COL_STATS_KEY);
if (globalColStats != null) {
- Map<String,CollectionStats> colStats = StatsUtil
- .colStatsMapFromString(globalColStats);
+ Map<String,CollectionStats> colStats = StatsUtil.colStatsMapFromString(globalColStats);
if (colStats != null) {
for (Entry<String,CollectionStats> e : colStats.entrySet()) {
addToGlobalColStats(req, e);
@@ -284,8 +277,7 @@ public class ExactStatsCache extends Sta
}
LOG.debug("Global collection stats={}", globalColStats);
if (globalTermStats == null) return;
- Map<String,TermStats> termStats = StatsUtil
- .termStatsMapFromString(globalTermStats);
+ Map<String,TermStats> termStats = StatsUtil.termStatsMapFromString(globalTermStats);
if (termStats != null) {
for (Entry<String,TermStats> e : termStats.entrySet()) {
addToGlobalTermStats(req, e);
@@ -294,10 +286,10 @@ public class ExactStatsCache extends Sta
}
protected void addToGlobalColStats(SolrQueryRequest req,
- Entry<String,CollectionStats> e) {
+ Entry<String,CollectionStats> e) {
Map<String,CollectionStats> currentGlobalColStats = (Map<String,CollectionStats>) req.getContext().get(CURRENT_GLOBAL_COL_STATS);
if (currentGlobalColStats == null) {
- currentGlobalColStats = new HashMap<String,CollectionStats>();
+ currentGlobalColStats = new HashMap<>();
req.getContext().put(CURRENT_GLOBAL_COL_STATS, currentGlobalColStats);
}
currentGlobalColStats.put(e.getKey(), e.getValue());
@@ -306,22 +298,22 @@ public class ExactStatsCache extends Sta
protected void addToGlobalTermStats(SolrQueryRequest req, Entry<String,TermStats> e) {
Map<String,TermStats> currentGlobalTermStats = (Map<String,TermStats>) req.getContext().get(CURRENT_GLOBAL_TERM_STATS);
if (currentGlobalTermStats == null) {
- currentGlobalTermStats = new HashMap<String,TermStats>();
+ currentGlobalTermStats = new HashMap<>();
req.getContext().put(CURRENT_GLOBAL_TERM_STATS, currentGlobalTermStats);
}
currentGlobalTermStats.put(e.getKey(), e.getValue());
}
-
+
protected static class ExactStatsSource extends StatsSource {
private final Map<String,TermStats> termStatsCache;
private final Map<String,CollectionStats> colStatsCache;
-
+
public ExactStatsSource(Map<String,TermStats> termStatsCache,
- Map<String,CollectionStats> colStatsCache) {
+ Map<String,CollectionStats> colStatsCache) {
this.termStatsCache = termStatsCache;
this.colStatsCache = colStatsCache;
}
-
+
public TermStatistics termStatistics(SolrIndexSearcher localSearcher, Term term, TermContext context)
throws IOException {
TermStats termStats = termStatsCache.get(term.toString());
@@ -335,7 +327,7 @@ public class ExactStatsCache extends Sta
return termStats.toTermStatistics();
}
}
-
+
@Override
public CollectionStatistics collectionStatistics(SolrIndexSearcher localSearcher, String field)
throws IOException {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java?rev=1694193&r1=1694192&r2=1694193&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/LRUStatsCache.java Wed Aug 5 12:07:30 2015
@@ -54,8 +54,7 @@ import org.slf4j.LoggerFactory;
* that is updated with the global statistics on every request.
*/
public class LRUStatsCache extends ExactStatsCache {
- private static final Logger LOG = LoggerFactory
- .getLogger(LRUStatsCache.class);
+ private static final Logger LOG = LoggerFactory.getLogger(LRUStatsCache.class);
// local stats obtained from shard servers
private final Map<String,SolrCache<String,TermStats>> perShardTermStats = new ConcurrentHashMap<>();
@@ -88,21 +87,18 @@ public class LRUStatsCache extends Exact
}
@Override
- protected void addToPerShardColStats(SolrQueryRequest req, String shard,
- Map<String,CollectionStats> colStats) {
+ protected void addToPerShardColStats(SolrQueryRequest req, String shard, Map<String,CollectionStats> colStats) {
perShardColStats.put(shard, colStats);
}
@Override
- protected Map<String,CollectionStats> getPerShardColStats(ResponseBuilder rb,
- String shard) {
+ protected Map<String,CollectionStats> getPerShardColStats(ResponseBuilder rb, String shard) {
return perShardColStats.get(shard);
}
@Override
protected void addToPerShardTermStats(SolrQueryRequest req, String shard, String termStatsString) {
- Map<String,TermStats> termStats = StatsUtil
- .termStatsMapFromString(termStatsString);
+ Map<String,TermStats> termStats = StatsUtil.termStatsMapFromString(termStatsString);
if (termStats != null) {
SolrCache<String,TermStats> cache = perShardTermStats.get(shard);
if (cache == null) { // initialize
@@ -119,12 +115,11 @@ public class LRUStatsCache extends Exact
@Override
protected TermStats getPerShardTermStats(SolrQueryRequest req, String t, String shard) {
SolrCache<String,TermStats> cache = perShardTermStats.get(shard);
- return cache.get(t);
+ return (cache != null) ? cache.get(t) : null; //Term doesn't exist in shard
}
@Override
- protected void addToGlobalColStats(SolrQueryRequest req,
- Entry<String,CollectionStats> e) {
+ protected void addToGlobalColStats(SolrQueryRequest req, Entry<String,CollectionStats> e) {
currentGlobalColStats.put(e.getKey(), e.getValue());
}
@@ -137,8 +132,7 @@ public class LRUStatsCache extends Exact
private final SolrCache<String,TermStats> termStatsCache;
private final Map<String,CollectionStats> colStatsCache;
- public LRUStatsSource(SolrCache<String,TermStats> termStatsCache,
- Map<String,CollectionStats> colStatsCache) {
+ public LRUStatsSource(SolrCache<String,TermStats> termStatsCache, Map<String,CollectionStats> colStatsCache) {
this.termStatsCache = termStatsCache;
this.colStatsCache = colStatsCache;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/StatsUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/StatsUtil.java?rev=1694193&r1=1694192&r2=1694193&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/StatsUtil.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/stats/StatsUtil.java Wed Aug 5 12:07:30 2015
@@ -195,7 +195,7 @@ public class StatsUtil {
if (data == null || data.trim().length() == 0) {
return null;
}
- Map<String,TermStats> map = new HashMap<String,TermStats>();
+ Map<String,TermStats> map = new HashMap<>();
String[] entries = data.split("!");
for (String es : entries) {
TermStats termStats = termStatsFromString(es, null);
Modified: lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml?rev=1694193&r1=1694192&r2=1694193&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/configsets/configset-2/conf/schema.xml Wed Aug 5 12:07:30 2015
@@ -18,8 +18,12 @@
<schema name="minimal" version="1.1">
<types>
<fieldType name="string" class="solr.StrField"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
</types>
<fields>
+ <field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="_root_" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
<dynamicField name="*" type="string" indexed="true" stored="true" />
</fields>
+ <uniqueKey>id</uniqueKey>
</schema>