You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jd...@apache.org on 2013/04/01 19:45:06 UTC
svn commit: r1463220 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/test/org/apache/solr/handler/component/
Author: jdyer
Date: Mon Apr 1 17:45:06 2013
New Revision: 1463220
URL: http://svn.apache.org/r1463220
Log:
SOLR-3758: SpellCheckComponent to work properly with distributed grouping
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1463220&r1=1463219&r2=1463220&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Mon Apr 1 17:45:06 2013
@@ -109,6 +109,9 @@ Bug Fixes
* SOLR-4641: Schema now throws exception on illegal field parameters. (Robert Muir)
+* SOLR-3758: Fixed SpellCheckComponent to work consistently with distributed grouping
+ (James Dyer)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java?rev=1463220&r1=1463219&r2=1463220&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java Mon Apr 1 17:45:06 2013
@@ -250,9 +250,9 @@ public class SpellCheckComponent extends
@Override
public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest sreq) {
SolrParams params = rb.req.getParams();
- // Turn on spellcheck only only when retrieving fields
if (!params.getBool(COMPONENT_NAME, false)) return;
- if ((sreq.purpose & ShardRequest.PURPOSE_GET_TOP_IDS) != 0) {
+ int purpose = rb.grouping() ? ShardRequest.PURPOSE_GET_TOP_GROUPS : ShardRequest.PURPOSE_GET_TOP_IDS;
+ if ((sreq.purpose & purpose) != 0) {
// fetch at least 5 suggestions from each shard
int count = sreq.params.getInt(SPELLCHECK_COUNT, 1);
if (count < 5) count = 5;
@@ -287,7 +287,7 @@ public class SpellCheckComponent extends
}
}
- long hits = rb.getNumberDocumentsFound();
+ long hits = rb.grouping() ? rb.totalHitCount : rb.getNumberDocumentsFound();
boolean isCorrectlySpelled = hits > (maxResultsForSuggest==null ? 0 : maxResultsForSuggest);
SpellCheckMergeData mergeData = new SpellCheckMergeData();
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java?rev=1463220&r1=1463219&r2=1463220&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java Mon Apr 1 17:45:06 2013
@@ -17,8 +17,11 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
import junit.framework.Assert;
-import junit.framework.TestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.BaseDistributedSearchTestCase;
@@ -39,11 +42,9 @@ import org.junit.BeforeClass;
@Slow
public class DistributedSpellCheckComponentTest extends BaseDistributedSearchTestCase {
- private String requestHandlerName;
- private String reqHandlerWithWordbreak;
-
public DistributedSpellCheckComponentTest()
{
+ //Helpful for debugging
//fixShardCount=true;
//shardCount=2;
//stress=0;
@@ -56,13 +57,6 @@ public class DistributedSpellCheckCompon
@Override
public void setUp() throws Exception {
- if(random().nextBoolean()) {
- requestHandlerName = "spellCheckCompRH";
- reqHandlerWithWordbreak = "spellCheckWithWordbreak";
- } else {
- requestHandlerName = "spellCheckCompRH_Direct";
- reqHandlerWithWordbreak = "spellCheckWithWordbreak_Direct";
- }
super.setUp();
}
@@ -90,9 +84,11 @@ public class DistributedSpellCheckCompon
@Override
public void validateControlData(QueryResponse control) throws Exception
{
- NamedList nl = control.getResponse();
- NamedList sc = (NamedList) nl.get("spellcheck");
- NamedList sug = (NamedList) sc.get("suggestions");
+ NamedList<Object> nl = control.getResponse();
+ @SuppressWarnings("unchecked")
+ NamedList<Object> sc = (NamedList<Object>) nl.get("spellcheck");
+ @SuppressWarnings("unchecked")
+ NamedList<Object> sug = (NamedList<Object>) sc.get("suggestions");
if(sug.size()==0) {
Assert.fail("Control data did not return any suggestions.");
}
@@ -133,21 +129,98 @@ public class DistributedSpellCheckCompon
handle.put("maxScore", SKIPVAL);
// we care only about the spellcheck results
handle.put("response", SKIP);
-
- q("q", "*:*", "spellcheck", "true", SpellingParams.SPELLCHECK_BUILD, "true", "qt", "spellCheckCompRH", "shards.qt", "spellCheckCompRH");
+ handle.put("grouped", SKIP);
- query("q", "*:*", "fl", "id,lowerfilt", "spellcheck.q","toyata", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName);
- query("q", "*:*", "fl", "id,lowerfilt", "spellcheck.q","toyata", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName, SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true");
- query("q", "*:*", "fl", "id,lowerfilt", "spellcheck.q","bluo", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName, SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true", SpellingParams.SPELLCHECK_COUNT, "4");
- query("q", "The quick reb fox jumped over the lazy brown dogs", "fl", "id,lowerfilt", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName, SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true", SpellingParams.SPELLCHECK_COUNT, "4", SpellingParams.SPELLCHECK_COLLATE, "true");
-
- query("q", "lowerfilt:(+quock +reb)", "fl", "id,lowerfilt", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName, SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true", SpellingParams.SPELLCHECK_COUNT, "10", SpellingParams.SPELLCHECK_COLLATE, "true", SpellingParams.SPELLCHECK_MAX_COLLATION_TRIES, "10", SpellingParams.SPELLCHECK_MAX_COLLATIONS, "10", SpellingParams.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true");
- query("q", "lowerfilt:(+quock +reb)", "fl", "id,lowerfilt", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName, SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true", SpellingParams.SPELLCHECK_COUNT, "10", SpellingParams.SPELLCHECK_COLLATE, "true", SpellingParams.SPELLCHECK_MAX_COLLATION_TRIES, "10", SpellingParams.SPELLCHECK_MAX_COLLATIONS, "10", SpellingParams.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "false");
- query("q", "lowerfilt:(+quock +reb)", "fl", "id,lowerfilt", "spellcheck", "true", "qt", requestHandlerName, "shards.qt", requestHandlerName, SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true", SpellingParams.SPELLCHECK_COUNT, "10", SpellingParams.SPELLCHECK_COLLATE, "true", SpellingParams.SPELLCHECK_MAX_COLLATION_TRIES, "0", SpellingParams.SPELLCHECK_MAX_COLLATIONS, "1", SpellingParams.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "false");
-
- query("q", "lowerfilt:(\"quote red fox\")", "fl", "id,lowerfilt", "spellcheck", "true", "qt", "spellCheckCompRH", "shards.qt", "spellCheckCompRH", SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_COUNT, "10", SpellCheckComponent.SPELLCHECK_COLLATE, "true", SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "10", SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1", SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_ALTERNATIVE_TERM_COUNT, "5", SpellCheckComponent.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST, "10");
- query("q", "lowerfilt:(\"rod fix\")", "fl", "id,lowerfilt", "spellcheck", "true", "qt", "spellCheckCompRH", "shards.qt", "spellCheckCompRH", SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_COUNT, "10", SpellCheckComponent.SPELLCHECK_COLLATE, "true", SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "10", SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1", SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_ALTERNATIVE_TERM_COUNT, "5", SpellCheckComponent.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST, "10");
-
- query("q", "lowerfilt:(+quock +redfox +jum +ped)", "fl", "id,lowerfilt", "spellcheck", "true", "qt", reqHandlerWithWordbreak, "shards.qt", reqHandlerWithWordbreak, SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true", SpellCheckComponent.SPELLCHECK_COUNT, "10", SpellCheckComponent.SPELLCHECK_COLLATE, "true", SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "0", SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1", SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true");
+ //Randomly select either IndexBasedSpellChecker or DirectSolrSpellChecker
+ String requestHandlerName = "spellCheckCompRH_Direct";
+ String reqHandlerWithWordbreak = "spellCheckWithWordbreak_Direct";
+ if(random().nextBoolean()) {
+ requestHandlerName = "spellCheckCompRH";
+ reqHandlerWithWordbreak = "spellCheckWithWordbreak";
+ }
+
+ //Shortcut names
+ String build = SpellingParams.SPELLCHECK_BUILD;
+ String extended = SpellingParams.SPELLCHECK_EXTENDED_RESULTS;
+ String count = SpellingParams.SPELLCHECK_COUNT;
+ String collate = SpellingParams.SPELLCHECK_COLLATE;
+ String collateExtended = SpellingParams.SPELLCHECK_COLLATE_EXTENDED_RESULTS;
+ String maxCollationTries = SpellingParams.SPELLCHECK_MAX_COLLATION_TRIES;
+ String maxCollations = SpellingParams.SPELLCHECK_MAX_COLLATIONS;
+ String altTermCount = SpellingParams.SPELLCHECK_ALTERNATIVE_TERM_COUNT;
+ String maxResults = SpellingParams.SPELLCHECK_MAX_RESULTS_FOR_SUGGEST;
+
+ //Build the dictionary for IndexBasedSpellChecker
+ q(buildRequest("*:*", false, "spellCheckCompRH", false, build, "true"));
+
+ //Test Basic Functionality
+ query(buildRequest("toyata", true, requestHandlerName, random().nextBoolean(), (String[]) null));
+ query(buildRequest("toyata", true, requestHandlerName, random().nextBoolean(), extended, "true"));
+ query(buildRequest("bluo", true, requestHandlerName, random().nextBoolean(), extended, "true", count, "4"));
+
+ //Test Collate functionality
+ query(buildRequest("The quick reb fox jumped over the lazy brown dogs",
+ false, requestHandlerName, random().nextBoolean(), extended, "true", count, "4", collate, "true"));
+ query(buildRequest("lowerfilt:(+quock +reb)",
+ false, requestHandlerName, random().nextBoolean(), extended, "true", count, "10",
+ collate, "true", maxCollationTries, "10", maxCollations, "10", collateExtended, "true"));
+ query(buildRequest("lowerfilt:(+quock +reb)",
+ false, requestHandlerName, random().nextBoolean(), extended, "true", count, "10",
+ collate, "true", maxCollationTries, "10", maxCollations, "10", collateExtended, "false"));
+ query(buildRequest("lowerfilt:(+quock +reb)",
+ false, requestHandlerName, random().nextBoolean(), extended, "true", count, "10",
+ collate, "true", maxCollationTries, "0", maxCollations, "1", collateExtended, "false"));
+
+ //Test context-sensitive collate
+ query(buildRequest("lowerfilt:(\"quote red fox\")",
+ false, requestHandlerName, random().nextBoolean(), extended, "true", count, "10",
+ collate, "true", maxCollationTries, "10", maxCollations, "1", collateExtended, "false",
+ altTermCount, "5", maxResults, "10"));
+ query(buildRequest("lowerfilt:(\"rod fix\")",
+ false, requestHandlerName, random().nextBoolean(), extended, "true", count, "10",
+ collate, "true", maxCollationTries, "10", maxCollations, "1", collateExtended, "false",
+ altTermCount, "5", maxResults, "10"));
+
+ //Test word-break spellchecker
+ query(buildRequest("lowerfilt:(+quock +redfox +jum +ped)",
+ false, reqHandlerWithWordbreak, random().nextBoolean(), extended, "true", count, "10",
+ collate, "true", maxCollationTries, "0", maxCollations, "1", collateExtended, "true"));
}
+ private Object[] buildRequest(String q, boolean useSpellcheckQ, String handlerName, boolean useGrouping, String... addlParams) {
+ List<Object> params = new ArrayList<Object>();
+
+ params.add("q");
+ params.add(useSpellcheckQ ? "*:*" : q);
+
+ if(useSpellcheckQ) {
+ params.add("spellcheck.q");
+ params.add(q);
+ }
+
+ params.add("fl");
+ params.add("id,lowerfilt");
+
+ params.add("qt");
+ params.add(handlerName);
+
+ params.add("shards.qt");
+ params.add(handlerName);
+
+ params.add("spellcheck");
+ params.add("true");
+
+ if(useGrouping) {
+ params.add("group");
+ params.add("true");
+
+ params.add("group.field");
+ params.add("id");
+ }
+
+ if(addlParams!=null) {
+ params.addAll(Arrays.asList(addlParams));
+ }
+ return params.toArray(new Object[params.size()]);
+ }
+
}