You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by nk...@apache.org on 2016/02/08 23:36:22 UTC
[47/50] [abbrv] lucene-solr git commit: SOLR-8496: multi-select
faceting and getDocSet(List) can match deleted docs
SOLR-8496: multi-select faceting and getDocSet(List<Query>) can match deleted docs
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene_solr_5_4@1725010 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7b42653a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7b42653a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7b42653a
Branch: refs/heads/branch_5_4
Commit: 7b42653a274962f50661f4bed52dad298f7064d5
Parents: e49d088
Author: Yonik Seeley <yo...@apache.org>
Authored: Sat Jan 16 17:34:12 2016 +0000
Committer: Yonik Seeley <yo...@apache.org>
Committed: Sat Jan 16 17:34:12 2016 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 12 ++++++++++++
.../apache/solr/search/SolrIndexSearcher.java | 19 +++++++++++++++----
.../solr/search/facet/TestJsonFacets.java | 20 ++++++++++++++++++--
3 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b42653a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6610e3a..1e10117 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -26,6 +26,12 @@ Bug Fixes
* SOLR-7462: AIOOBE in RecordingJSONParser (Scott Dawson, noble)
+* SOLR-8496: SolrIndexSearcher.getDocSet(List<Query>) incorrectly included deleted documents
+ when all of the queries were uncached (or there was no filter cache). This caused
+ multi-select faceting (including the JSON Facet API) to include deleted doc counts
+ when the remaining non-excluded filters were all uncached. This bug was first introduced in 5.3.0
+ (Andreas Müller, Vasiliy Bout, Erick Erickson, Shawn Heisey, Hossman, yonik)
+
New Features
----------------------
@@ -512,6 +518,12 @@ Bug Fixes
* SOLR-7462: AIOOBE in RecordingJSONParser (Scott Dawson, noble)
+* SOLR-8496: SolrIndexSearcher.getDocSet(List<Query>) incorrectly included deleted documents
+ when all of the queries were uncached (or there was no filter cache). This caused
+ multi-select faceting (including the JSON Facet API) to include deleted doc counts
+ when the remaining non-excluded filters were all uncached. This bug was first introduced in 5.3.0
+ (Andreas Müller, Vasiliy Bout, Erick Erickson, Shawn Heisey, Hossman, yonik)
+
New Features
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b42653a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 0c65eff..280fa85 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -953,6 +953,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
public DocSet answer; // the answer, if non-null
public Filter filter;
public DelegatingCollector postFilter;
+ public boolean hasDeletedDocs; // true if it's possible that filter may match deleted docs
}
@@ -1018,7 +1019,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
for (final LeafReaderContext leaf : leafContexts) {
final LeafReader reader = leaf.reader();
- final Bits liveDocs = reader.getLiveDocs(); // TODO: the filter may already only have liveDocs...
+ Bits liveDocs = reader.getLiveDocs();
DocIdSet idSet = null;
if (pf.filter != null) {
idSet = pf.filter.getDocIdSet(leaf, liveDocs);
@@ -1028,21 +1029,30 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
if (idSet != null) {
idIter = idSet.iterator();
if (idIter == null) continue;
+ if (!pf.hasDeletedDocs) liveDocs = null; // no need to check liveDocs
}
final LeafCollector leafCollector = collector.getLeafCollector(leaf);
int max = reader.maxDoc();
if (idIter == null) {
- for (int docid = 0; docid<max; docid++) {
+ for (int docid = 0; docid < max; docid++) {
if (liveDocs != null && !liveDocs.get(docid)) continue;
leafCollector.collect(docid);
}
} else {
- for (int docid = -1; (docid = idIter.advance(docid+1)) < max; ) {
- leafCollector.collect(docid);
+ if (liveDocs != null) {
+ for (int docid = -1; (docid = idIter.advance(docid + 1)) < max; ) {
+ if (liveDocs.get(docid))
+ leafCollector.collect(docid);
+ }
+ } else {
+ for (int docid = -1; (docid = idIter.advance(docid + 1)) < max;) {
+ leafCollector.collect(docid);
+ }
}
}
+
}
if(collector instanceof DelegatingCollector) {
@@ -1144,6 +1154,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
weights.add(createNormalizedWeight(qq, true));
}
pf.filter = new FilterImpl(answer, weights);
+ pf.hasDeletedDocs = (answer == null); // if all clauses were uncached, the resulting filter may match deleted docs
} else {
if (postFilters == null) {
if (answer == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b42653a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index d4c1c00..685b599 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -382,7 +382,11 @@ public class TestJsonFacets extends SolrTestCaseHS {
client.deleteByQuery("*:*", null);
- client.add(sdoc("id", "1", cat_s, "A", where_s, "NY", num_d, "4", num_i, "2", super_s, "zodiac", date,"2001-01-01T01:01:01Z", val_b, "true", sparse_s, "one"), null);
+ SolrInputDocument doc =
+ sdoc("id", "1", cat_s, "A", where_s, "NY", num_d, "4", num_i, "2", super_s, "zodiac", date, "2001-01-01T01:01:01Z", val_b, "true", sparse_s, "one");
+ client.add(doc, null);
+ client.add(doc, null);
+ client.add(doc, null); // a couple of deleted docs
client.add(sdoc("id", "2", cat_s, "B", where_s, "NJ", num_d, "-9", num_i, "-5", super_s,"superman", date,"2002-02-02T02:02:02Z", val_b, "false" , multi_ss,"a", multi_ss,"b" , Z_num_i, "0"), null);
client.add(sdoc("id", "3"), null);
client.commit();
@@ -917,6 +921,18 @@ public class TestJsonFacets extends SolrTestCaseHS {
// multi-select / exclude tagged filters via excludeTags
////////////////////////////////////////////////////////////////////////////////////////////
+ // test uncached multi-select (see SOLR-8496)
+ client.testJQ(params(p, "q", "{!cache=false}*:*", "fq","{!tag=doc3,allfilt}-id:3"
+
+ , "json.facet", "{" +
+ "f1:{${terms} type:terms, field:${cat_s}, domain:{excludeTags:doc3} } " +
+ "}"
+ )
+ , "facets=={ count:5, " +
+ " f1:{ buckets:[ {val:B, count:3}, {val:A, count:2} ] }" +
+ "}"
+ );
+
// nested query facets on subset (with excludeTags)
client.testJQ(params(p, "q", "*:*", "fq","{!tag=abc}id:(2 3)"
, "json.facet", "{ processEmpty:true," +
@@ -941,7 +957,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
);
// terms facet with nested query facet (with excludeTags, using new format inside domain:{})
- client.testJQ(params(p, "q", "*:*", "fq", "{!tag=doc6,allfilt}-id:6", "fq","{!tag=doc3,allfilt}-id:3"
+ client.testJQ(params(p, "q", "{!cache=false}*:*", "fq", "{!tag=doc6,allfilt}-id:6", "fq","{!tag=doc3,allfilt}-id:3"
, "json.facet", "{processEmpty:true, " +
" f0:{${terms} type:terms, field:${cat_s}, facet:{nj:{query:'${where_s}:NJ'}} } " +