You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/12/13 01:13:16 UTC
svn commit: r1645099 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/test/org/apache/solr/handler/component/
Author: shalin
Date: Sat Dec 13 00:13:15 2014
New Revision: 1645099
URL: http://svn.apache.org/r1645099
Log:
SOLR-6604: SOLR-6812: Fix NPE with distrib.singlePass=true and expand component. Increased test coverage of expand component with docValues.
This closes #98.
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1645099&r1=1645098&r2=1645099&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Sat Dec 13 00:13:15 2014
@@ -285,6 +285,10 @@ Bug Fixes
* SOLR-6626: NPE in FieldMutatingUpdateProcessor when indexing a doc with
null field value (Noble Paul)
+* SOLR-6604: SOLR-6812: Fix NPE with distrib.singlePass=true and expand
+ component. Increased test coverage of expand component with docValues.
+ (Christine Poerschke, Per Steffensen, shalin)
+
Optimizations
----------------------
@@ -442,12 +446,6 @@ Other Changes
the example directory instead of server/solr. (Alexandre Rafalovitch, Anshum Gupta, hossman,
Timothy Potter)
-* SOLR-6843: JMX RMI connector should be disabled by default but can be activated by
- setting ENABLE_REMOTE_JMX_OPTS to true in solr.in.(sh|cmd). (Timothy Potter)
-
-* SOLR-6844: Rename ConfigSolr.getZkHostPort(), which actually returns the Solr port,
- to .getSolrHostPort(). (Martijn Koster, Steve Rowe)
-
================== 4.10.3 ==================
Bug Fixes
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java?rev=1645099&r1=1645098&r2=1645099&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java Sat Dec 13 00:13:15 2014
@@ -86,6 +86,7 @@ import com.carrotsearch.hppc.cursors.Obj
*/
public class ExpandComponent extends SearchComponent implements PluginInfoInitialized, SolrCoreAware {
public static final String COMPONENT_NAME = "expand";
+ private static final int finishingStage = ResponseBuilder.STAGE_GET_FIELDS;
private PluginInfo info = PluginInfo.EMPTY_INFO;
@Override
@@ -116,13 +117,6 @@ public class ExpandComponent extends Sea
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
- boolean isShard = params.getBool(ShardParams.IS_SHARD, false);
- String ids = params.get(ShardParams.IDS);
-
- if (ids == null && isShard) {
- return;
- }
-
String field = params.get(ExpandParams.EXPAND_FIELD);
if (field == null) {
List<Query> filters = rb.getFilters();
@@ -247,8 +241,22 @@ public class ExpandComponent extends Sea
}
@Override
+ public int distributedProcess(ResponseBuilder rb) throws IOException {
+ if (rb.doExpand && rb.stage < finishingStage) {
+ return finishingStage;
+ }
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ @Override
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
-
+ SolrParams params = rb.req.getParams();
+ if (!params.getBool(COMPONENT_NAME, false)) return;
+ if (!rb.onePassDistributedQuery && (sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) == 0) {
+ sreq.params.set(COMPONENT_NAME, "false");
+ } else {
+ sreq.params.set(COMPONENT_NAME, "true");
+ }
}
@SuppressWarnings("unchecked")
@@ -286,7 +294,7 @@ public class ExpandComponent extends Sea
return;
}
- if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
+ if (rb.stage != finishingStage) {
return;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java?rev=1645099&r1=1645098&r2=1645099&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java Sat Dec 13 00:13:15 2014
@@ -50,20 +50,22 @@ public class DistributedExpandComponentT
@Override
public void doTest() throws Exception {
+ final String group = (random().nextBoolean() ? "group_s" : "group_s_dv");
+
del("*:*");
- index_specific(0,"id","1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000");
- index_specific(0,"id","2", "term_s", "YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200");
- index_specific(1,"id","5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000");
- index_specific(1,"id","6", "term_s", "YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200");
- index_specific(0,"id","7", "term_s", "YYYY", "group_s", "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000");
- index_specific(1,"id","8", "term_s", "YYYY", "group_s", "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200");
- index_specific(2,"id","9", "term_s", "YYYY", "group_s", "group3", "test_ti", "1000", "test_tl", "1005", "test_tf", "3000");
- index_specific(2, "id", "10", "term_s", "YYYY", "group_s", "group3", "test_ti", "1500", "test_tl", "1001", "test_tf", "3200");
- index_specific(2,"id", "11", "term_s", "YYYY", "group_s", "group3", "test_ti", "1300", "test_tl", "1002", "test_tf", "3300");
- index_specific(1,"id","12", "term_s", "YYYY", "group_s", "group4", "test_ti", "15", "test_tl", "10", "test_tf", "2000");
- index_specific(1,"id","13", "term_s", "YYYY", "group_s", "group4", "test_ti", "16", "test_tl", "9", "test_tf", "2000");
- index_specific(1,"id","14", "term_s", "YYYY", "group_s", "group4", "test_ti", "1", "test_tl", "20", "test_tf", "2000");
+ index_specific(0,"id","1", "term_s", "YYYY", group, "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000");
+ index_specific(0,"id","2", "term_s", "YYYY", group, "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200");
+ index_specific(1,"id","5", "term_s", "YYYY", group, "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000");
+ index_specific(1,"id","6", "term_s", "YYYY", group, "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200");
+ index_specific(0,"id","7", "term_s", "YYYY", group, "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000");
+ index_specific(1,"id","8", "term_s", "YYYY", group, "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200");
+ index_specific(2,"id","9", "term_s", "YYYY", group, "group3", "test_ti", "1000", "test_tl", "1005", "test_tf", "3000");
+ index_specific(2, "id", "10", "term_s", "YYYY", group, "group3", "test_ti", "1500", "test_tl", "1001", "test_tf", "3200");
+ index_specific(2,"id", "11", "term_s", "YYYY", group, "group3", "test_ti", "1300", "test_tl", "1002", "test_tf", "3300");
+ index_specific(1,"id","12", "term_s", "YYYY", group, "group4", "test_ti", "15", "test_tl", "10", "test_tf", "2000");
+ index_specific(1,"id","13", "term_s", "YYYY", group, "group4", "test_ti", "16", "test_tl", "9", "test_tf", "2000");
+ index_specific(1,"id","14", "term_s", "YYYY", group, "group4", "test_ti", "1", "test_tl", "20", "test_tf", "2000");
commit();
@@ -80,21 +82,21 @@ public class DistributedExpandComponentT
handle.put("maxScore", SKIPVAL);
handle.put("_version_", SKIP);
- query("q", "*:*", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "fl","*,score");
- query("q", "*:*", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "fl","*,score");
- query("q", "*:*", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
+ query("q", "*:*", "fq", "{!collapse field="+group+"}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "fl","*,score");
+ query("q", "*:*", "fq", "{!collapse field="+group+"}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "fl","*,score");
+ query("q", "*:*", "fq", "{!collapse field="+group+"}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
//Test no expand results
- query("q", "test_ti:5", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
+ query("q", "test_ti:5", "fq", "{!collapse field="+group+"}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
//Test zero results
- query("q", "test_ti:5434343", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
+ query("q", "test_ti:5434343", "fq", "{!collapse field="+group+"}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
//Test page 2
- query("q", "*:*", "start","1", "rows", "1", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "fl","*,score");
+ query("q", "*:*", "start","1", "rows", "1", "fq", "{!collapse field="+group+"}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "fl","*,score");
//First basic test case.
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -113,7 +115,7 @@ public class DistributedExpandComponentT
params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -132,7 +134,7 @@ public class DistributedExpandComponentT
params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -147,6 +149,45 @@ public class DistributedExpandComponentT
assertExpandGroupCountAndOrder("group3", 1, results, "9.0");
assertExpandGroupCountAndOrder("group4", 1, results, "14.0");
+
+ //Test key-only fl
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field="+group+"}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("fl", "id");
+
+ setDistributedParams(params);
+ rsp = queryServer(params);
+ results = rsp.getExpandedResults();
+ assertExpandGroups(results, "group1","group2", "group3", "group4");
+ assertExpandGroupCountAndOrder("group1", 2, results, "1.0", "7.0");
+ assertExpandGroupCountAndOrder("group2", 2, results, "5.0", "8.0");
+ assertExpandGroupCountAndOrder("group3", 2, results, "11.0", "9.0");
+ assertExpandGroupCountAndOrder("group4", 2, results, "12.0", "14.0");
+
+ //Test distrib.singlePass true
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field="+group+"}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("distrib.singlePass", "true");
+
+ setDistributedParams(params);
+ rsp = queryServer(params);
+ results = rsp.getExpandedResults();
+ assertExpandGroups(results, "group1","group2", "group3", "group4");
+ assertExpandGroupCountAndOrder("group1", 2, results, "1.0", "7.0");
+ assertExpandGroupCountAndOrder("group2", 2, results, "5.0", "8.0");
+ assertExpandGroupCountAndOrder("group3", 2, results, "11.0", "9.0");
+ assertExpandGroupCountAndOrder("group4", 2, results, "12.0", "14.0");
+
}
private void assertExpandGroups(Map<String, SolrDocumentList> expandedResults, String... groups) throws Exception {
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java?rev=1645099&r1=1645098&r2=1645099&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java Sat Dec 13 00:13:15 2014
@@ -45,10 +45,12 @@ public class TestExpandComponent extends
@Test
public void testExpand() throws Exception {
- String[] doc = {"id","1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
+ final String group = (random().nextBoolean() ? "group_s" : "group_s_dv");
+
+ String[] doc = {"id","1", "term_s", "YYYY", group, "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
assertU(adoc(doc));
assertU(commit());
- String[] doc1 = {"id","2", "term_s","YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200", "type_s", "child"};
+ String[] doc1 = {"id","2", "term_s","YYYY", group, "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200", "type_s", "child"};
assertU(adoc(doc1));
String[] doc2 = {"id","3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"};
@@ -58,17 +60,17 @@ public class TestExpandComponent extends
assertU(adoc(doc3));
- String[] doc4 = {"id","5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
+ String[] doc4 = {"id","5", "term_s", "YYYY", group, "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
assertU(adoc(doc4));
assertU(commit());
- String[] doc5 = {"id","6", "term_s","YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200", "type_s", "child"};
+ String[] doc5 = {"id","6", "term_s","YYYY", group, "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200", "type_s", "child"};
assertU(adoc(doc5));
assertU(commit());
- String[] doc6 = {"id","7", "term_s", "YYYY", "group_s", "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000", "type_s", "child"};
+ String[] doc6 = {"id","7", "term_s", "YYYY", group, "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000", "type_s", "child"};
assertU(adoc(doc6));
assertU(commit());
- String[] doc7 = {"id","8", "term_s","YYYY", "group_s", "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200", "type_s", "child"};
+ String[] doc7 = {"id","8", "term_s","YYYY", group, "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200", "type_s", "child"};
assertU(adoc(doc7));
assertU(commit());
@@ -76,7 +78,7 @@ public class TestExpandComponent extends
//First basic test case.
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -94,7 +96,7 @@ public class TestExpandComponent extends
params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -110,7 +112,7 @@ public class TestExpandComponent extends
//Test expand.sort
params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -129,7 +131,7 @@ public class TestExpandComponent extends
//Main result set should include the doc with null value in the collapse field.
params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s nullPolicy=collapse}");
+ params.add("fq", "{!collapse field="+group+" nullPolicy=collapse}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -154,7 +156,7 @@ public class TestExpandComponent extends
params.add("bf", "field(test_ti)");
params.add("expand", "true");
params.add("expand.q", "type_s:child");
- params.add("expand.field", "group_s");
+ params.add("expand.field", group);
params.add("expand.sort", "test_tl desc");
assertQ(req(params), "*[count(/response/result/doc)=2]",
"*[count(/response/lst[@name='expanded']/result)=2]",
@@ -176,7 +178,7 @@ public class TestExpandComponent extends
params.add("bf", "field(test_ti)");
params.add("expand", "true");
params.add("expand.fq", "type_s:child");
- params.add("expand.field", "group_s");
+ params.add("expand.field", group);
params.add("expand.sort", "test_tl desc");
assertQ(req(params), "*[count(/response/result/doc)=2]",
"*[count(/response/lst[@name='expanded']/result)=2]",
@@ -198,7 +200,7 @@ public class TestExpandComponent extends
params.add("expand", "true");
params.add("expand.q", "type_s:child");
params.add("expand.fq", "*:*");
- params.add("expand.field", "group_s");
+ params.add("expand.field", group);
params.add("expand.sort", "test_tl desc");
assertQ(req(params), "*[count(/response/result/doc)=2]",
"*[count(/response/lst[@name='expanded']/result)=2]",
@@ -214,7 +216,7 @@ public class TestExpandComponent extends
params = new ModifiableSolrParams();
params.add("q", "*:*");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -235,7 +237,7 @@ public class TestExpandComponent extends
params = new ModifiableSolrParams();
params.add("q", "test_ti:5");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -249,7 +251,7 @@ public class TestExpandComponent extends
params = new ModifiableSolrParams();
params.add("q", "test_ti:5532535");
- params.add("fq", "{!collapse field=group_s}");
+ params.add("fq", "{!collapse field="+group+"}");
params.add("defType", "edismax");
params.add("bf", "field(test_ti)");
params.add("expand", "true");
@@ -258,6 +260,25 @@ public class TestExpandComponent extends
assertQ(req(params), "*[count(/response/result/doc)=0]",
"*[count(/response/lst[@name='expanded']/result)=0]"
);
+
+ //Test key-only fl
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field="+group+"}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("fl", "id");
+ assertQ(req(params), "*[count(/response/result/doc)=2]",
+ "*[count(/response/lst[@name='expanded']/result)=2]",
+ "/response/result/doc[1]/float[@name='id'][.='2.0']",
+ "/response/result/doc[2]/float[@name='id'][.='6.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='1.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='7.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='5.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='8.0']"
+ );
}
}