You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/12/10 19:05:09 UTC
svn commit: r1212830 [2/2] - in /lucene/dev/branches/lucene3622: ./ lucene/
lucene/contrib/ lucene/src/java/org/apache/lucene/index/
lucene/src/test/org/apache/lucene/index/ solr/ solr/contrib/
solr/contrib/clustering/ solr/contrib/clustering/src/java/...
Modified: lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java (original)
+++ lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/EchoClusteringAlgorithm.java Sat Dec 10 18:05:08 2011
@@ -15,6 +15,7 @@ package org.apache.solr.handler.clusteri
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import java.util.Collections;
import java.util.List;
import org.carrot2.core.Cluster;
@@ -48,6 +49,12 @@ public class EchoClusteringAlgorithm ext
@Attribute(key = AttributeNames.CLUSTERS)
private List<Cluster> clusters;
+ @Input
+ @Processing
+ @Attribute(key = "custom-fields")
+ private String customFields = "";
+
+
@Override
public void process() throws ProcessingException {
clusters = Lists.newArrayListWithCapacity(documents.size());
@@ -55,6 +62,15 @@ public class EchoClusteringAlgorithm ext
for (Document document : documents) {
final Cluster cluster = new Cluster();
cluster.addPhrases(document.getTitle(), document.getSummary());
+ if (document.getLanguage() != null) {
+ cluster.addPhrases(document.getLanguage().name());
+ }
+ for (String field : customFields.split(",")) {
+ Object value = document.getField(field);
+ if (value != null) {
+ cluster.addPhrases(value.toString());
+ }
+ }
cluster.addDocuments(document);
clusters.add(cluster);
}
Modified: lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java (original)
+++ lucene/dev/branches/lucene3622/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/LexicalResourcesCheckClusteringAlgorithm.java Sat Dec 10 18:05:08 2011
@@ -25,9 +25,7 @@ import org.carrot2.core.ProcessingCompon
import org.carrot2.core.ProcessingException;
import org.carrot2.core.attribute.AttributeNames;
import org.carrot2.core.attribute.Processing;
-import org.carrot2.text.linguistic.DefaultLexicalDataFactory;
import org.carrot2.text.linguistic.ILexicalData;
-import org.carrot2.text.linguistic.ILexicalDataFactory;
import org.carrot2.text.preprocessing.pipeline.BasicPreprocessingPipeline;
import org.carrot2.text.util.MutableCharArray;
import org.carrot2.util.attribute.Attribute;
@@ -46,37 +44,37 @@ import com.google.common.collect.Lists;
*/
@Bindable(prefix = "LexicalResourcesCheckClusteringAlgorithm")
public class LexicalResourcesCheckClusteringAlgorithm extends
- ProcessingComponentBase implements IClusteringAlgorithm {
+ ProcessingComponentBase implements IClusteringAlgorithm {
- @Output
- @Processing
- @Attribute(key = AttributeNames.CLUSTERS)
- private List<Cluster> clusters;
-
- @Input
- @Processing
- @Attribute
- private String wordsToCheck;
-
- private BasicPreprocessingPipeline preprocessing = new BasicPreprocessingPipeline();
-
- @Override
- public void process() throws ProcessingException {
- clusters = Lists.newArrayList();
- if (wordsToCheck == null) {
- return;
- }
-
- // Test with Maltese so that the English clustering performed in other tests
- // is not affected by the test stopwords and stoplabels.
- ILexicalData lexicalData = preprocessing.lexicalDataFactory
- .getLexicalData(LanguageCode.MALTESE);
-
- for (String word : wordsToCheck.split(",")) {
- if (!lexicalData.isCommonWord(new MutableCharArray(word))
- && !lexicalData.isStopLabel(word)) {
- clusters.add(new Cluster(word));
- }
- }
- }
+ @Output
+ @Processing
+ @Attribute(key = AttributeNames.CLUSTERS)
+ private List<Cluster> clusters;
+
+ @Input
+ @Processing
+ @Attribute
+ private String wordsToCheck;
+
+ private BasicPreprocessingPipeline preprocessing = new BasicPreprocessingPipeline();
+
+ @Override
+ public void process() throws ProcessingException {
+ clusters = Lists.newArrayList();
+ if (wordsToCheck == null) {
+ return;
+ }
+
+ // Test with Maltese so that the English clustering performed in other tests
+ // is not affected by the test stopwords and stoplabels.
+ ILexicalData lexicalData = preprocessing.lexicalDataFactory
+ .getLexicalData(LanguageCode.MALTESE);
+
+ for (String word : wordsToCheck.split(",")) {
+ if (!lexicalData.isCommonWord(new MutableCharArray(word))
+ && !lexicalData.isStopLabel(word)) {
+ clusters.add(new Cluster(word));
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java (original)
+++ lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java Sat Dec 10 18:05:08 2011
@@ -17,6 +17,7 @@
package org.apache.solr.handler.dataimport;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
@@ -27,8 +28,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
-import java.util.Map;
-import java.util.Set;
/**
* <p> Writes documents to SOLR. </p>
@@ -43,12 +42,14 @@ public class SolrWriter extends DIHWrite
static final String LAST_INDEX_KEY = "last_index_time";
private final UpdateRequestProcessor processor;
-
+ private final int commitWithin;
+
SolrQueryRequest req;
public SolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
this.processor = processor;
this.req = req;
+ commitWithin = (req != null) ? req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1): -1;
}
@Override
@@ -65,6 +66,7 @@ public class SolrWriter extends DIHWrite
try {
AddUpdateCommand command = new AddUpdateCommand(req);
command.solrDoc = d;
+ command.commitWithin = commitWithin;
processor.processAdd(command);
} catch (Exception e) {
log.warn("Error creating document : " + d, e);
Modified: lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java (original)
+++ lucene/dev/branches/lucene3622/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java Sat Dec 10 18:05:08 2011
@@ -24,6 +24,7 @@ import org.apache.solr.client.solrj.resp
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.UpdateParams;
import org.junit.After;
import org.junit.Before;
@@ -80,6 +81,33 @@ public class TestContentStreamDataSource
assertEquals("Hello C1", ((List)doc.getFieldValue("desc")).get(0));
}
+ @Test
+ public void testCommitWithin() throws Exception {
+ DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
+ ModifiableSolrParams params = params("command", "full-import",
+ "clean", "false", UpdateParams.COMMIT, "false",
+ UpdateParams.COMMIT_WITHIN, "1000");
+ req.setParams(params);
+ String url = "http://localhost:" + jetty.getLocalPort() + "/solr";
+ CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(url);
+ solrServer.request(req);
+ Thread.sleep(100);
+ ModifiableSolrParams queryAll = params("q", "*");
+ QueryResponse qres = solrServer.query(queryAll);
+ SolrDocumentList results = qres.getResults();
+ assertEquals(0, results.getNumFound());
+ Thread.sleep(1000);
+ for (int i = 0; i < 10; i++) {
+ qres = solrServer.query(queryAll);
+ results = qres.getResults();
+ if (2 == results.getNumFound()) {
+ return;
+ }
+ Thread.sleep(500);
+ }
+ fail("Commit should have occured but it did not");
+ }
+
private class SolrInstance {
String name;
Integer port;
Modified: lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/lucene3622/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Sat Dec 10 18:05:08 2011
@@ -162,13 +162,18 @@ public class QueryComponent extends Sear
//TODO: move weighting of sort
Sort groupSort = searcher.weightSort(cmd.getSort());
- // groupSort defaults to sort
- String groupSortStr = params.get(GroupParams.GROUP_SORT);
if (groupSort == null) {
- groupSort = new Sort();
+ groupSort = Sort.RELEVANCE;
}
+
+ // groupSort defaults to sort
+ String groupSortStr = params.get(GroupParams.GROUP_SORT);
//TODO: move weighting of sort
Sort sortWithinGroup = groupSortStr == null ? groupSort : searcher.weightSort(QueryParsing.parseSort(groupSortStr, req));
+ if (sortWithinGroup == null) {
+ sortWithinGroup = Sort.RELEVANCE;
+ }
+
groupingSpec.setSortWithinGroup(sortWithinGroup);
groupingSpec.setGroupSort(groupSort);
Modified: lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java?rev=1212830&r1=1212829&r2=1212830&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java (original)
+++ lucene/dev/branches/lucene3622/solr/core/src/test/org/apache/solr/TestDistributedGrouping.java Sat Dec 10 18:05:08 2011
@@ -140,6 +140,7 @@ public class TestDistributedGrouping ext
query("q", "*:*", "fq", s1 + ":a", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " asc, id asc", "group.truncate", "true", "facet", "true", "facet.field", t1);
// We cannot validate distributed grouping with scoring as first sort. since there is no global idf. We can check if no errors occur
+ simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", i1 + " desc", "group.sort", "score desc"); // SOLR-2955
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", "score desc, _docid_ asc, id asc");
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.field", i1, "group.limit", 10);
}
@@ -149,6 +150,7 @@ public class TestDistributedGrouping ext
for (int i = 0; i < queryParams.length; i += 2) {
params.add(queryParams[i].toString(), queryParams[i + 1].toString());
}
+ params.set("shards", shards);
queryServer(params);
}