You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by st...@apache.org on 2011/12/09 16:58:42 UTC

svn commit: r1212489 - in /lucene/dev/trunk/solr/contrib/clustering: ./ src/java/org/apache/solr/handler/clustering/carrot2/ src/test-files/clustering/solr/conf/ src/test/org/apache/solr/handler/clustering/ src/test/org/apache/solr/handler/clustering/c...

Author: stanislaw
Date: Fri Dec  9 15:58:42 2011
New Revision: 1212489

URL: http://svn.apache.org/viewvc?rev=1212489&view=rev
Log:
SOLR-2938: Clustering on multiple fields

Modified:
    lucene/dev/trunk/solr/contrib/clustering/CHANGES.txt
    lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
    lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml
    lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java
    lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java

Modified: lucene/dev/trunk/solr/contrib/clustering/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/CHANGES.txt?rev=1212489&r1=1212488&r2=1212489&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/contrib/clustering/CHANGES.txt Fri Dec  9 15:58:42 2011
@@ -18,6 +18,10 @@ $Id$
   by the clustering plugin, can be overridden by carrot.summarySnippets
   if needed (Stanislaw Osinski).
 
+* SOLR-2938: Clustering on multiple fields. The carrot.title and 
+  carrot.snippet can now take comma- or space-separated lists of
+  field names to cluster (Stanislaw Osinski).
+
 ================== Release 3.5.0 ==================
 
 (No Changes)

Modified: lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java?rev=1212489&r1=1212488&r2=1212489&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java Fri Dec  9 15:58:42 2011
@@ -19,6 +19,7 @@ package org.apache.solr.handler.clusteri
 
 import java.io.*;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -294,13 +295,17 @@ public class CarrotClusteringEngine exte
   private Set<String> getFieldsForClustering(SolrQueryRequest sreq) {
     SolrParams solrParams = sreq.getParams();
 
-    String titleField = solrParams.get(CarrotParams.TITLE_FIELD_NAME, "title");
-    String snippetField = solrParams.get(CarrotParams.SNIPPET_FIELD_NAME, titleField);
-    if (StringUtils.isBlank(snippetField)) {
+    String titleFieldSpec = solrParams.get(CarrotParams.TITLE_FIELD_NAME, "title");
+    String snippetFieldSpec = solrParams.get(CarrotParams.SNIPPET_FIELD_NAME, titleFieldSpec);
+    if (StringUtils.isBlank(snippetFieldSpec)) {
       throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, CarrotParams.SNIPPET_FIELD_NAME
               + " must not be blank.");
     }
-    return Sets.newHashSet(titleField, snippetField);
+    
+    final Set<String> fields = Sets.newHashSet();
+    fields.addAll(Arrays.asList(titleFieldSpec.split("[, ]")));
+    fields.addAll(Arrays.asList(snippetFieldSpec.split("[, ]")));
+    return fields;
   }
 
   /**
@@ -313,8 +318,8 @@ public class CarrotClusteringEngine exte
     SolrCore core = sreq.getCore();
 
     String urlField = solrParams.get(CarrotParams.URL_FIELD_NAME, "url");
-    String titleField = solrParams.get(CarrotParams.TITLE_FIELD_NAME, "title");
-    String snippetField = solrParams.get(CarrotParams.SNIPPET_FIELD_NAME, titleField);
+    String titleFieldSpec = solrParams.get(CarrotParams.TITLE_FIELD_NAME, "title");
+    String snippetFieldSpec = solrParams.get(CarrotParams.SNIPPET_FIELD_NAME, titleFieldSpec);
     
     // Get the documents
     boolean produceSummary = solrParams.getBool(CarrotParams.PRODUCE_SUMMARY, false);
@@ -325,7 +330,7 @@ public class CarrotClusteringEngine exte
       highlighter = HighlightComponent.getHighlighter(core);
       if (highlighter != null){
         Map<String, Object> args = Maps.newHashMap();
-        snippetFieldAry = new String[]{snippetField};
+        snippetFieldAry = snippetFieldSpec.split("[, ]");
         args.put(HighlightParams.FIELDS, snippetFieldAry);
         args.put(HighlightParams.HIGHLIGHT, "true");
         args.put(HighlightParams.SIMPLE_PRE, ""); //we don't care about actually highlighting the area
@@ -353,7 +358,8 @@ public class CarrotClusteringEngine exte
 
     while (docsIter.hasNext()) {
       SolrDocument sdoc = docsIter.next();
-      String snippet = getValue(sdoc, snippetField);
+      String snippet = null;
+      
       // TODO: docIds will be null when running distributed search.
       // See comment in ClusteringComponent#finishStage().
       if (produceSummary && docIds != null) {
@@ -361,24 +367,32 @@ public class CarrotClusteringEngine exte
         DocList docAsList = new DocSlice(0, 1, docsHolder, scores, 1, 1.0f);
         NamedList<Object> highlights = highlighter.doHighlighting(docAsList, theQuery, req, snippetFieldAry);
         if (highlights != null && highlights.size() == 1) {//should only be one value given our setup
-          //should only be one document with one field
+          //should only be one document
           @SuppressWarnings("unchecked")
           NamedList<String []> tmp = (NamedList<String[]>) highlights.getVal(0);
-          String [] highlt = tmp.get(snippetField);
           
-          // Join fragments with a period, so that Carrot2 does not create
-          // cross-fragment phrases, such phrases rarely make sense.
-          if (highlt != null && highlt.length > 0) {
-            final StringBuilder sb = new StringBuilder(highlt[0]);
-            for (int i = 1; i < highlt.length; i++) {
-              sb.append(" . ");
-              sb.append(highlt[i]);
+          final StringBuilder sb = new StringBuilder();
+          for (int j = 0; j < snippetFieldAry.length; j++) {
+            // Join fragments with a period, so that Carrot2 does not create
+            // cross-fragment phrases, such phrases rarely make sense.
+            String [] highlt = tmp.get(snippetFieldAry[j]);
+            if (highlt != null && highlt.length > 0) {
+              for (int i = 0; i < highlt.length; i++) {
+                sb.append(highlt[i]);
+                sb.append(" . ");
+              }
             }
-            snippet = sb.toString();
           }
+          snippet = sb.toString();
         }
       }
-      Document carrotDocument = new Document(getValue(sdoc, titleField),
+      
+      // If summaries not enabled or summary generation failed, use full content.
+      if (snippet == null) {
+        snippet = getConcatenated(sdoc, snippetFieldSpec);
+      }
+      
+      Document carrotDocument = new Document(getConcatenated(sdoc, titleFieldSpec),
               snippet, (String)sdoc.getFieldValue(urlField));
       carrotDocument.setField(SOLR_DOCUMENT_ID, sdoc.getFieldValue(idFieldName));
       result.add(carrotDocument);
@@ -387,16 +401,18 @@ public class CarrotClusteringEngine exte
     return result;
   }
 
-  protected String getValue(SolrDocument sdoc, String field) {
+  private String getConcatenated(SolrDocument sdoc, String fieldsSpec) {
     StringBuilder result = new StringBuilder();
-    Collection<Object> vals = sdoc.getFieldValues(field);
-    if(vals == null) return "";
-    Iterator<Object> ite = vals.iterator();
-    while(ite.hasNext()){
-      // Join multiple values with a period so that Carrot2 does not pick up
-      // phrases that cross field value boundaries (in most cases it would
-      // create useless phrases).
-      result.append((String)ite.next()).append(" . ");
+    for (String field : fieldsSpec.split("[, ]")) {
+      Collection<Object> vals = sdoc.getFieldValues(field);
+      if (vals == null) continue;
+      Iterator<Object> ite = vals.iterator();
+      while(ite.hasNext()){
+        // Join multiple values with a period so that Carrot2 does not pick up
+        // phrases that cross field value boundaries (in most cases it would
+        // create useless phrases).
+        result.append((String)ite.next()).append(" . ");
+      }
     }
     return result.toString().trim();
   }

Modified: lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml?rev=1212489&r1=1212488&r2=1212489&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/schema.xml Fri Dec  9 15:58:42 2011
@@ -282,6 +282,7 @@
    <field name="url" type="string" indexed="true" stored="true" required="true" />
 
    <field name="title" type="text" indexed="true" stored="true" multiValued="true"/>
+   <field name="heading" type="text" indexed="true" stored="true" multiValued="true"/>
    <field name="snippet" type="text" indexed="true" stored="true" multiValued="true"/>
    <field name="body" type="text" indexed="true" stored="true" multiValued="true"/>
    <!-- catchall field, containing all other searchable text fields (implemented

Modified: lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java?rev=1212489&r1=1212488&r2=1212489&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java Fri Dec  9 15:58:42 2011
@@ -46,6 +46,16 @@ public abstract class AbstractClustering
     multiValuedSnippet.addField("snippet", "Third value of multi field. Some more text. And still more.");
     assertNull(h.validateUpdate(adoc(multiValuedSnippet)));
 
+    // Add a document with multi-field title and snippet
+    final SolrInputDocument multiFieldDoc = new SolrInputDocument();
+    multiFieldDoc.addField("id", numberOfDocs++);
+    multiFieldDoc.addField("title", "Title field");
+    multiFieldDoc.addField("heading", "Heading field");
+    multiFieldDoc.addField("url", "URL");
+    multiFieldDoc.addField("snippet", "Snippet field: this is the contents of the snippet field.");
+    multiFieldDoc.addField("body", "Body field: this is the contents of the body field that will get clustered together with snippet.");
+    assertNull(h.validateUpdate(adoc(multiFieldDoc)));
+    
     assertNull(h.validateUpdate(commit()));
   }
 

Modified: lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java?rev=1212489&r1=1212488&r2=1212489&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java Fri Dec  9 15:58:42 2011
@@ -88,8 +88,8 @@ public class CarrotClusteringEngineTest 
 
   private List<NamedList<Object>> clusterWithHighlighting(
       boolean enableHighlighting, int fragSize) throws IOException {
-    // Two documents don't have mining in the snippet
-    return clusterWithHighlighting(enableHighlighting, fragSize, 1, "mine", numberOfDocs - 2);
+    // Some documents don't have mining in the snippet
+    return clusterWithHighlighting(enableHighlighting, fragSize, 1, "mine", numberOfDocs - 4);
   }
 
   private List<NamedList<Object>> clusterWithHighlighting(
@@ -253,6 +253,47 @@ public class CarrotClusteringEngineTest 
     assertTrue("Summary covers second value", snippetWithSummary.contains("Second"));
     assertTrue("Summary covers third value", snippetWithSummary.contains("Third"));
   }
+  
+  @Test
+  public void concatenatingMultipleFields() throws Exception {
+    final ModifiableSolrParams params = new ModifiableSolrParams();
+    params.add(CarrotParams.TITLE_FIELD_NAME, "title,heading");
+    params.add(CarrotParams.SNIPPET_FIELD_NAME, "snippet,body");
+
+    final List<String> labels = getLabels(checkEngine(
+        getClusteringEngine("echo"), 1, 1, new TermQuery(new Term("body",
+            "snippet")), params).get(0));
+    assertTrue("Snippet contains third value", labels.get(0).contains("Title field"));
+    assertTrue("Snippet contains third value", labels.get(0).contains("Heading field"));
+    assertTrue("Snippet contains third value", labels.get(1).contains("Snippet field"));
+    assertTrue("Snippet contains third value", labels.get(1).contains("Body field"));
+  }
+
+  @Test
+  public void highlightingMultipleFields() throws Exception {
+    final TermQuery query = new TermQuery(new Term("snippet", "content"));
+
+    final ModifiableSolrParams params = new ModifiableSolrParams();
+    params.add(CarrotParams.TITLE_FIELD_NAME, "title,heading");
+    params.add(CarrotParams.SNIPPET_FIELD_NAME, "snippet,body");
+    params.add(CarrotParams.PRODUCE_SUMMARY, Boolean.toString(false));
+    
+    final String snippetWithoutSummary = getLabels(checkEngine(
+        getClusteringEngine("echo"), 1, 1, query, params).get(0)).get(1);
+    assertTrue("Snippet covers snippet field", snippetWithoutSummary.contains("snippet field"));
+    assertTrue("Snippet covers body field", snippetWithoutSummary.contains("body field"));
+
+    params.set(CarrotParams.PRODUCE_SUMMARY, Boolean.toString(true));
+    params.add(CarrotParams.SUMMARY_FRAGSIZE, Integer.toString(30));
+    params.add(CarrotParams.SUMMARY_SNIPPETS, Integer.toString(2));
+    final String snippetWithSummary = getLabels(checkEngine(
+        getClusteringEngine("echo"), 1, 1, query, params).get(0)).get(1);    
+    assertTrue("Snippet with summary shorter than full snippet",
+        snippetWithoutSummary.length() > snippetWithSummary.length());
+    assertTrue("Snippet covers snippet field", snippetWithSummary.contains("snippet field"));
+    assertTrue("Snippet covers body field", snippetWithSummary.contains("body field"));
+
+  }
 
   private CarrotClusteringEngine getClusteringEngine(String engineName) {
     ClusteringComponent comp = (ClusteringComponent) h.getCore()