You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/12 10:15:38 UTC
svn commit: r1034304 [6/7] - in /lucene/dev/branches/docvalues: ./ lucene/
lucene/contrib/ lucene/contrib/benchmark/conf/
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/contrib/benchmark/src/test/org/apache/lucene/be...
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java Fri Nov 12 09:15:30 2010
@@ -125,6 +125,15 @@ public class SolrWriter {
}
}
+ void finish() {
+ try {
+ processor.finish();
+ } catch (IOException e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to call finish() on UpdateRequestProcessor", e);
+ }
+ }
+
Properties readIndexerProperties() {
Properties props = new Properties();
InputStream propInput = null;
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java Fri Nov 12 09:15:30 2010
@@ -277,7 +277,7 @@ public class XPathRecordReader {
if (event == END_ELEMENT) {
if (flattenedStarts > 0) flattenedStarts--;
else {
- if (text.length() > 0 && valuesAddedinThisFrame != null) {
+ if (hasText && valuesAddedinThisFrame != null) {
valuesAddedinThisFrame.add(fieldName);
putText(values, text.toString(), fieldName, multiValued);
}
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/webapp/admin/debug.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/webapp/admin/debug.jsp?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/webapp/admin/debug.jsp (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/webapp/admin/debug.jsp Fri Nov 12 09:15:30 2010
@@ -20,10 +20,10 @@
<html>
<head>
<title>DataImportHandler Interactive Development</title>
-<link rel="stylesheet" type="text/css" href="solr-admin.css">
-<link rel="icon" href="favicon.ico" type="image/ico"></link>
-<link rel="shortcut icon" href="favicon.ico" type="image/ico"></link>
-<script src="jquery-1.2.3.min.js"></script>
+<link rel="stylesheet" type="text/css" href="solr-admin.css"/>
+<link rel="icon" href="favicon.ico" type="image/ico"/>
+<link rel="shortcut icon" href="favicon.ico" type="image/ico"/>
+<script src="jquery-1.4.3.min.js"></script>
</head>
<body>
<h1>DataImportHandler Development Console</h1>
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java Fri Nov 12 09:15:30 2010
@@ -19,6 +19,15 @@ package org.apache.solr.handler.dataimpo
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.update.MergeIndexesCommand;
+import org.apache.solr.update.RollbackUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before;
@@ -270,4 +279,69 @@ public abstract class AbstractDataImport
return delegate.replaceTokens(template);
}
}
+
+ public static class TestUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory {
+
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return new TestUpdateRequestProcessor(next);
+ }
+
+ }
+
+ public static class TestUpdateRequestProcessor extends UpdateRequestProcessor {
+
+ public static boolean finishCalled = false;
+ public static boolean processAddCalled = false;
+ public static boolean processCommitCalled = false;
+ public static boolean processDeleteCalled = false;
+ public static boolean mergeIndexesCalled = false;
+ public static boolean rollbackCalled = false;
+
+ public static void reset() {
+ finishCalled = false;
+ processAddCalled = false;
+ processCommitCalled = false;
+ processDeleteCalled = false;
+ mergeIndexesCalled = false;
+ rollbackCalled = false;
+ }
+
+ public TestUpdateRequestProcessor(UpdateRequestProcessor next) {
+ super(next);
+ reset();
+ }
+
+ public void finish() throws IOException {
+ finishCalled = true;
+ super.finish();
+ }
+
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ processAddCalled = true;
+ super.processAdd(cmd);
+ }
+
+ public void processCommit(CommitUpdateCommand cmd) throws IOException {
+ processCommitCalled = true;
+ super.processCommit(cmd);
+ }
+
+ public void processDelete(DeleteUpdateCommand cmd) throws IOException {
+ processDeleteCalled = true;
+ super.processDelete(cmd);
+ }
+
+ public void processMergeIndexes(MergeIndexesCommand cmd) throws IOException {
+ mergeIndexesCalled = true;
+ super.processMergeIndexes(cmd);
+ }
+
+ public void processRollback(RollbackUpdateCommand cmd) throws IOException {
+ rollbackCalled = true;
+ super.processRollback(cmd);
+ }
+
+ }
}
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java Fri Nov 12 09:15:30 2010
@@ -60,6 +60,7 @@ public class TestDocBuilder extends Abst
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
assertEquals(0, swi.docs.size());
assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
@@ -81,6 +82,7 @@ public class TestDocBuilder extends Abst
di.runCmd(rp, swi);
assertEquals(Boolean.FALSE, swi.deleteAllCalled);
assertEquals(Boolean.FALSE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
assertEquals(0, swi.docs.size());
assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
@@ -104,6 +106,7 @@ public class TestDocBuilder extends Abst
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
assertEquals(1, swi.docs.size());
assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
@@ -134,6 +137,7 @@ public class TestDocBuilder extends Abst
di.runCmd(rp, swi);
assertEquals(Boolean.FALSE, swi.deleteAllCalled);
assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
assertEquals(1, swi.docs.size());
assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
assertEquals(1, di.getDocBuilder().importStatistics.docCount.get());
@@ -168,6 +172,7 @@ public class TestDocBuilder extends Abst
di.runCmd(rp, swi);
assertEquals(Boolean.TRUE, swi.deleteAllCalled);
assertEquals(Boolean.TRUE, swi.commitCalled);
+ assertEquals(Boolean.TRUE, swi.finishCalled);
assertEquals(3, swi.docs.size());
for (int i = 0; i < l.size(); i++) {
Map<String, Object> map = (Map<String, Object>) l.get(i);
@@ -189,6 +194,8 @@ public class TestDocBuilder extends Abst
Boolean commitCalled = Boolean.FALSE;
+ Boolean finishCalled = Boolean.FALSE;
+
public SolrWriterImpl() {
super(null, ".");
}
@@ -208,6 +215,10 @@ public class TestDocBuilder extends Abst
public void commit(boolean b) {
commitCalled = Boolean.TRUE;
}
+
+ public void finish() {
+ finishCalled = Boolean.TRUE;
+ }
}
public static final String dc_singleEntity = "<dataConfig>\n"
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java Fri Nov 12 09:15:30 2010
@@ -52,6 +52,10 @@ public class TestDocBuilder2 extends Abs
runFullImport(loadDataConfig("single-entity-data-config.xml"));
assertQ(req("id:1"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor processCommit was not callled", TestUpdateRequestProcessor.processCommitCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
}
@Test
@@ -66,6 +70,8 @@ public class TestDocBuilder2 extends Abs
assertQ(req("id:1"), "//*[@numFound='1']");
assertTrue("Start event listener was not called", StartEventListener.executed);
assertTrue("End event listener was not called", EndEventListener.executed);
+ assertTrue("Update request processor processAdd was not called", TestUpdateRequestProcessor.processAddCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
}
@Test
@@ -200,6 +206,9 @@ public class TestDocBuilder2 extends Abs
assertQ(req("id:2"), "//*[@numFound='0']");
assertQ(req("id:3"), "//*[@numFound='1']");
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
MockDataSource.clearCache();
rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "one"));
@@ -212,6 +221,10 @@ public class TestDocBuilder2 extends Abs
assertQ(req("id:1"), "//*[@numFound='0']");
assertQ(req("id:2"), "//*[@numFound='0']");
assertQ(req("id:3"), "//*[@numFound='1']");
+
+ assertTrue("Update request processor processDelete was not called", TestUpdateRequestProcessor.processDeleteCalled);
+ assertTrue("Update request processor finish was not called", TestUpdateRequestProcessor.finishCalled);
+
}
@Test
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java Fri Nov 12 09:15:30 2010
@@ -195,6 +195,29 @@ public class TestXPathRecordReader exten
}
@Test
+ public void testElems2LevelEmpty() {
+ String xml="<root>\n"
+ + "\t<a>\n"
+ + "\t <b>\n\t <x>x0</x>\n"
+ + "\t <y>y0</y>\n"
+ + "\t </b>\n"
+ + "\t <b>\n\t <x></x>\n" // empty
+ + "\t <y>y1</y>\n"
+ + "\t </b>\n"
+ + "\t</a>\n"
+ + "</root>";
+ XPathRecordReader rr = new XPathRecordReader("/root/a");
+ rr.addField("a", "/root/a/b/x", true);
+ rr.addField("b", "/root/a/b/y", true);
+ List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
+ assertEquals(1, l.size());
+ assertEquals("x0",((List) l.get(0).get("a")).get(0));
+ assertEquals("y0",((List) l.get(0).get("b")).get(0));
+ assertEquals("",((List) l.get(0).get("a")).get(1));
+ assertEquals("y1",((List) l.get(0).get("b")).get(1));
+ }
+
+ @Test
public void testMixedContent() {
String xml = "<xhtml:p xmlns:xhtml=\"http://xhtml.com/\" >This text is \n" +
" <xhtml:b>bold</xhtml:b> and this text is \n" +
@@ -236,12 +259,16 @@ public class TestXPathRecordReader exten
public void testElems2LevelWithAttrib() {
String xml = "<root>\n\t<a>\n\t <b k=\"x\">\n"
+ "\t <x>x0</x>\n"
- + "\t <y>y0</y>\n"
+ + "\t <y></y>\n" // empty
+ "\t </b>\n"
+ "\t <b k=\"y\">\n"
- + "\t <x>x1</x>\n"
+ + "\t <x></x>\n" // empty
+ "\t <y>y1</y>\n"
+ "\t </b>\n"
+ + "\t <b k=\"z\">\n"
+ + "\t <x>x2</x>\n"
+ + "\t <y>y2</y>\n"
+ + "\t </b>\n"
+ "\t </a>\n"
+ "\t <a>\n\t <b>\n"
+ "\t <x>x3</x>\n"
@@ -256,8 +283,14 @@ public class TestXPathRecordReader exten
rr.addField("y", "/root/a/b[@k]/y", true);
List<Map<String, Object>> l = rr.getAllRecords(new StringReader(xml));
assertEquals(2, l.size());
- assertEquals(2, ((List) l.get(0).get("x")).size());
- assertEquals(2, ((List) l.get(0).get("y")).size());
+ assertEquals(3, ((List) l.get(0).get("x")).size());
+ assertEquals(3, ((List) l.get(0).get("y")).size());
+ assertEquals("x0", ((List) l.get(0).get("x")).get(0));
+ assertEquals("", ((List) l.get(0).get("y")).get(0));
+ assertEquals("", ((List) l.get(0).get("x")).get(1));
+ assertEquals("y1", ((List) l.get(0).get("y")).get(1));
+ assertEquals("x2", ((List) l.get(0).get("x")).get(2));
+ assertEquals("y2", ((List) l.get(0).get("y")).get(2));
assertEquals(0, l.get(1).size());
}
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/contentstream-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/contentstream-solrconfig.xml?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/contentstream-solrconfig.xml (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/contentstream-solrconfig.xml Fri Nov 12 09:15:30 2010
@@ -385,7 +385,7 @@
<str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
-->
</requestHandler>
-
+
<!-- config for the admin interface -->
<admin>
<defaultQuery>*:*</defaultQuery>
@@ -395,5 +395,11 @@
-->
</admin>
+ <updateRequestProcessorChain key="contentstream" default="true">
+ <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ <processor class="solr.LogUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
</config>
Modified: lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solrconfig.xml?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solrconfig.xml Fri Nov 12 09:15:30 2010
@@ -391,5 +391,11 @@
-->
</admin>
+ <updateRequestProcessorChain key="dataimport" default="true">
+ <processor class="org.apache.solr.handler.dataimport.AbstractDataImportHandlerTestCase$TestUpdateRequestProcessorFactory"/>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ <processor class="solr.LogUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
</config>
Propchange: lucene/dev/branches/docvalues/solr/example/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
/lucene/dev/branches/branch_3x/solr/example:949730,961612,979161,980654,982195,987811,988512
/lucene/dev/branches/preflexfixes/solr/example:967125-979432
-/lucene/dev/trunk/solr/example:1021634-1029001
+/lucene/dev/trunk/solr/example:1021634-1034284
/lucene/java/branches/lucene_2_4/solr/example:748824
/lucene/java/branches/lucene_2_9/solr/example:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr/example:818601-821336
Modified: lucene/dev/branches/docvalues/solr/example/example-DIH/solr/rss/conf/rss-data-config.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/example-DIH/solr/rss/conf/rss-data-config.xml?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/example-DIH/solr/rss/conf/rss-data-config.xml (original)
+++ lucene/dev/branches/docvalues/solr/example/example-DIH/solr/rss/conf/rss-data-config.xml Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
<dataConfig>
- <dataSource type="HttpDataSource" />
+ <dataSource type="URLDataSource" />
<document>
<entity name="slashdot"
pk="link"
Modified: lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/head.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/head.vm?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/head.vm (original)
+++ lucene/dev/branches/docvalues/solr/example/solr/conf/velocity/head.vm Fri Nov 12 09:15:30 2010
@@ -3,7 +3,7 @@
<title>#param('title')</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
-<script type="text/javascript" src="#{url_for_solr}/admin/jquery-1.2.3.min.js"></script>
+<script type="text/javascript" src="#{url_for_solr}/admin/jquery-1.4.3.min.js"></script>
<link rel="stylesheet" type="text/css" href="#{url_for_solr}/admin/file?file=/velocity/main.css&contentType=text/css"/>
<link rel="stylesheet" href="#{url_for_solr}/admin/file?file=/velocity/jquery.autocomplete.css&contentType=text/css" type="text/css" />
<script type="text/javascript" src="#{url_for_solr}/admin/file?file=/velocity/jquery.autocomplete.js&contentType=text/javascript"></script>
Propchange: lucene/dev/branches/docvalues/solr/lib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
/lucene/dev/branches/branch_3x/solr/lib:949730,961612,979161,980654,982195,987811,988512
/lucene/dev/branches/preflexfixes/solr/lib:967125-979432
-/lucene/dev/trunk/solr/lib:1021634-1029001
+/lucene/dev/trunk/solr/lib:1021634-1034284
/lucene/java/branches/lucene_2_4/solr/lib:748824
/lucene/java/branches/lucene_2_9/solr/lib:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr/lib:818601-821336
Propchange: lucene/dev/branches/docvalues/solr/lib/commons-httpclient-3.1.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar:949730,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/solr/lib/commons-httpclient-3.1.jar:949730,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/solr/lib/commons-httpclient-3.1.jar:967125-979432
-/lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar:1021634-1029001
+/lucene/dev/trunk/solr/lib/commons-httpclient-3.1.jar:1021634-1034284
/lucene/solr/branches/newtrunk/solr/lib/commons-httpclient-3.1.jar:924462
/lucene/solr/trunk/lib/commons-httpclient-3.1.jar:922950-923910,923912-925091
Propchange: lucene/dev/branches/docvalues/solr/lib/jcl-over-slf4j-1.5.5.jar
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar:949730,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/solr/lib/jcl-over-slf4j-1.5.5.jar:949730,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/solr/lib/jcl-over-slf4j-1.5.5.jar:967125-979432
-/lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar:1021634-1029001
+/lucene/dev/trunk/solr/lib/jcl-over-slf4j-1.5.5.jar:1021634-1034284
/lucene/solr/branches/newtrunk/solr/lib/jcl-over-slf4j-1.5.5.jar:924462
/lucene/solr/trunk/lib/jcl-over-slf4j-1.5.5.jar:922950-923910,923912-925091
Propchange: lucene/dev/branches/docvalues/solr/site/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
/lucene/dev/branches/branch_3x/solr/site:949730,961612,979161,980654,982195,987811,988512
/lucene/dev/branches/preflexfixes/solr/site:967125-979432
-/lucene/dev/trunk/solr/site:1021634-1029001
+/lucene/dev/trunk/solr/site:1021634-1034284
/lucene/java/branches/lucene_2_4/solr/site:748824
/lucene/java/branches/lucene_2_9/solr/site:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr/site:818601-821336
Propchange: lucene/dev/branches/docvalues/solr/src/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
/lucene/dev/branches/branch_3x/solr/src:949730,961612,979161,980654,982195,987811,988512
/lucene/dev/branches/preflexfixes/solr/src:967125-979432
-/lucene/dev/trunk/solr/src:1021634-1029001
+/lucene/dev/trunk/solr/src:1021634-1034284
/lucene/java/branches/lucene_2_4/solr/src:748824
/lucene/java/branches/lucene_2_9/solr/src:817269-818600,825998,829134,829881,831036,896850,909334
/lucene/java/branches/lucene_2_9_back_compat_tests/solr/src:818601-821336
Propchange: lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common:949730,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/solr/src/common/org/apache/solr/common:949730,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/solr/src/common/org/apache/solr/common:967125-979432
-/lucene/dev/trunk/solr/src/common/org/apache/solr/common:1021634-1029001
+/lucene/dev/trunk/solr/src/common/org/apache/solr/common:1021634-1034284
/lucene/solr/branches/newtrunk/solr/src/common/org/apache/solr/common:924462
/lucene/solr/trunk/src/common/org/apache/solr/common:922950-923910,923912-925091
Modified: lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/params/GroupParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/params/GroupParams.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/params/GroupParams.java (original)
+++ lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/params/GroupParams.java Fri Nov 12 09:15:30 2010
@@ -18,7 +18,7 @@
package org.apache.solr.common.params;
/**
- * Facet parameters
+ * Group parameters
*/
public interface GroupParams {
public static final String GROUP = "group";
@@ -30,5 +30,7 @@ public interface GroupParams {
/** the limit for the number of documents in each group */
public static final String GROUP_LIMIT = GROUP + ".limit";
+ /** the offset for the doclist of each group */
+ public static final String GROUP_OFFSET = GROUP + ".offset";
}
Modified: lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/util/StrUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/util/StrUtils.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/util/StrUtils.java (original)
+++ lucene/dev/branches/docvalues/solr/src/common/org/apache/solr/common/util/StrUtils.java Fri Nov 12 09:15:30 2010
@@ -239,6 +239,24 @@ public class StrUtils {
}
/**
+ * {@link NullPointerException} and {@link SolrException} free version of {@link #parseBool(String)}
+ * @param s
+ * @param def
+ * @return
+ */
+ public static boolean parseBool(String s, boolean def) {
+ if( s != null ) {
+ if( s.startsWith("true") || s.startsWith("on") || s.startsWith("yes") ) {
+ return true;
+ }
+ if( s.startsWith("false") || s.startsWith("off") || s.equals("no") ) {
+ return false;
+ }
+ }
+ return def;
+ }
+
+ /**
* URLEncodes a value, replacing only enough chars so that
* the URL may be unambiguously pasted back into a browser.
* <p>
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/HyphenationCompoundWordTokenFilterFactory.java Fri Nov 12 09:15:30 2010
@@ -17,10 +17,6 @@
package org.apache.solr.analysis;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.compound.CompoundWordTokenFilterBase;
@@ -33,6 +29,8 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.util.plugin.ResourceLoaderAware;
import java.util.Map;
+import java.io.InputStream;
+import org.xml.sax.InputSource;
/**
* Factory for {@link HyphenationCompoundWordTokenFilter}
@@ -57,7 +55,7 @@ public class HyphenationCompoundWordToke
private HyphenationTree hyphenator;
private String dictFile;
private String hypFile;
- private String encoding = "UTF-8"; // default to UTF-8 encoding
+ private String encoding;
private int minWordSize;
private int minSubwordSize;
private int maxSubwordSize;
@@ -82,18 +80,21 @@ public class HyphenationCompoundWordToke
}
public void inform(ResourceLoader loader) {
- Reader reader = null;
+ InputStream stream = null;
try {
if (dictFile != null) // the dictionary can be empty.
dictionary = getWordSet(loader, dictFile, false);
-
- InputStream hyph = loader.openResource(hypFile);
- reader = new InputStreamReader(hyph, encoding);
- hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(reader);
- } catch (Exception e) { // TODO: getHyphenationTree really shouldnt throw "Exception"
+ // TODO: Broken, because we cannot resolve real system id
+ // ResourceLoader should also supply method like ClassLoader to get resource URL
+ stream = loader.openResource(hypFile);
+ final InputSource is = new InputSource(stream);
+ is.setEncoding(encoding); // if it's null let xml parser decide
+ is.setSystemId(hypFile);
+ hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);
+ } catch (Exception e) { // TODO: getHyphenationTree really shouldn't throw "Exception"
throw new RuntimeException(e);
} finally {
- IOUtils.closeQuietly(reader);
+ IOUtils.closeQuietly(stream);
}
}
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/PivotFacetHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/PivotFacetHelper.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/PivotFacetHelper.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/PivotFacetHelper.java Fri Nov 12 09:15:30 2010
@@ -17,6 +17,8 @@
package org.apache.solr.handler.component;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.DocSet;
import org.apache.solr.common.SolrException;
@@ -95,10 +97,8 @@ public class PivotFacetHelper
{
SolrIndexSearcher searcher = rb.req.getSearcher();
// TODO: optimize to avoid converting to an external string and then having to convert back to internal below
- FieldType ftype = searcher.getSchema().getField(field).getType();
-
- // Required to translate back to an object
- Field f = new Field( field, "X", Store.YES, Index.ANALYZED );
+ SchemaField sfield = searcher.getSchema().getField(field);
+ FieldType ftype = sfield.getType();
String nextField = fnames.poll();
@@ -106,19 +106,21 @@ public class PivotFacetHelper
for (Map.Entry<String, Integer> kv : superFacets) {
// Only sub-facet if parent facet has positive count - still may not be any values for the sub-field though
if (kv.getValue() > minMatch ) {
- String internal = ftype.toInternal( kv.getKey() );
- f.setValue( internal );
+ // don't reuse the same BytesRef each time since we will be constructing Term
+ // objects that will most likely be cached.
+ BytesRef termval = new BytesRef();
+ ftype.readableToIndexed(kv.getKey(), termval);
SimpleOrderedMap<Object> pivot = new SimpleOrderedMap<Object>();
pivot.add( "field", field );
- pivot.add( "value", ftype.toObject( f ) );
+ pivot.add( "value", ftype.toObject(sfield, termval) );
pivot.add( "count", kv.getValue() );
if( subField == null ) {
values.add( pivot );
}
else {
- Query query = new TermQuery(new Term(field, internal));
+ Query query = new TermQuery(new Term(field, termval));
DocSet subset = searcher.getDocSet(query, docs);
SimpleFacets sf = getFacetImplementation(rb.req, subset, rb.req.getParams());
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/component/QueryComponent.java Fri Nov 12 09:15:30 2010
@@ -300,8 +300,8 @@ public class QueryComponent extends Sear
boolean doGroup = params.getBool(GroupParams.GROUP, false);
if (doGroup) {
try {
- cmd.groupCommands = new ArrayList<Grouping.Command>();
-
+ Grouping grouping = new Grouping(searcher, result, cmd);
+
String[] fields = params.getParams(GroupParams.GROUP_FIELD);
String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
@@ -311,6 +311,7 @@ public class QueryComponent extends Sear
Sort groupSort = groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req) : null;
int limitDefault = cmd.getLen(); // this is normally from "rows"
+ int groupOffsetDefault = params.getInt(GroupParams.GROUP_OFFSET, 0);
int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
// temporary: implement all group-by-field as group-by-func
@@ -329,7 +330,7 @@ public class QueryComponent extends Sear
for (String groupByStr : funcs) {
QParser parser = QParser.getParser(groupByStr, "func", rb.req);
Query q = parser.getQuery();
- Grouping.CommandFunc gc = new Grouping.CommandFunc();
+ Grouping.CommandFunc gc = grouping.new CommandFunc();
gc.groupSort = groupSort;
if (q instanceof FunctionQuery) {
@@ -340,8 +341,11 @@ public class QueryComponent extends Sear
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
+ gc.groupOffset = groupOffsetDefault;
+ gc.offset = cmd.getOffset();
+ gc.sort = cmd.getSort();
- cmd.groupCommands.add(gc);
+ grouping.add(gc);
}
}
@@ -349,33 +353,31 @@ public class QueryComponent extends Sear
for (String groupByStr : queries) {
QParser parser = QParser.getParser(groupByStr, null, rb.req);
Query gq = parser.getQuery();
- Grouping.CommandQuery gc = new Grouping.CommandQuery();
+ Grouping.CommandQuery gc = grouping.new CommandQuery();
gc.query = gq;
gc.groupSort = groupSort;
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
+ gc.groupOffset = groupOffsetDefault;
- cmd.groupCommands.add(gc);
+ grouping.add(gc);
}
}
- if (cmd.groupCommands.size() == 0)
- cmd.groupCommands = null;
+ if (rb.doHighlights || rb.isDebug()) {
+ // we need a single list of the returned docs
+ cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
+ }
- if (cmd.groupCommands != null) {
- if (rb.doHighlights || rb.isDebug()) {
- // we need a single list of the returned docs
- cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
- }
+ // searcher.search(result,cmd);
+ grouping.execute();
+ rb.setResult( result );
+ rsp.add("grouped", result.groupedResults);
+ // TODO: get "hits" a different way to log
+ return;
- searcher.search(result,cmd);
- rb.setResult( result );
- rsp.add("grouped", result.groupedResults);
- // TODO: get "hits" a different way to log
- return;
- }
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/BoolField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/BoolField.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/BoolField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/BoolField.java Fri Nov 12 09:15:30 2010
@@ -109,6 +109,11 @@ public class BoolField extends FieldType
return Boolean.valueOf( toExternal(f) );
}
+ @Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return term.bytes[0] == 'T';
+ }
+
public String indexedToReadable(String indexedForm) {
char ch = indexedForm.charAt(0);
return ch=='T' ? "true" : "false";
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/FieldType.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/FieldType.java Fri Nov 12 09:15:30 2010
@@ -346,6 +346,13 @@ public abstract class FieldType extends
return toExternal(f); // by default use the string
}
+ public Object toObject(SchemaField sf, BytesRef term) {
+ CharArr ext = new CharArr(term.length);
+ indexedToReadable(term, ext);
+ Field f = createField(sf, ext.toString(), 1.0f);
+ return toObject(f);
+ }
+
/** Given an indexed term, return the human readable representation */
public String indexedToReadable(String indexedForm) {
return indexedForm;
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/StrField.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/StrField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/StrField.java Fri Nov 12 09:15:30 2010
@@ -28,6 +28,7 @@ import org.apache.solr.search.function.F
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.StringIndexDocValues;
import org.apache.solr.search.QParser;
+import org.apache.solr.util.ByteUtils;
import java.util.Map;
import java.io.IOException;
@@ -54,64 +55,11 @@ public class StrField extends FieldType
public ValueSource getValueSource(SchemaField field, QParser parser) {
return new StrFieldSource(field.getName());
}
-}
-
-
-class StrFieldSource extends FieldCacheSource {
-
- public StrFieldSource(String field) {
- super(field);
- }
- public String description() {
- return "str(" + field + ')';
- }
-
- public DocValues getValues(Map context, IndexReader reader) throws IOException {
- return new StringIndexDocValues(this, reader, field) {
- protected String toTerm(String readableValue) {
- return readableValue;
- }
-
- public float floatVal(int doc) {
- return (float)intVal(doc);
- }
-
- public int intVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- return ord;
- }
-
- public long longVal(int doc) {
- return (long)intVal(doc);
- }
-
- public double doubleVal(int doc) {
- return (double)intVal(doc);
- }
-
- public String strVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord == 0) {
- return null;
- } else {
- return termsIndex.lookup(ord, new BytesRef()).utf8ToString();
- }
- }
-
- public String toString(int doc) {
- return description() + '=' + strVal(doc);
- }
- };
+ @Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return ByteUtils.UTF8toUTF16(term);
}
+}
- public boolean equals(Object o) {
- return o instanceof StrFieldSource
- && super.equals(o);
- }
- private static int hcode = SortableFloatFieldSource.class.hashCode();
- public int hashCode() {
- return hcode + super.hashCode();
- };
-}
\ No newline at end of file
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TextField.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TextField.java Fri Nov 12 09:15:30 2010
@@ -31,9 +31,11 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.util.BytesRef;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.response.XMLWriter;
import org.apache.solr.search.QParser;
+import org.apache.solr.util.ByteUtils;
import java.util.Map;
import java.util.List;
@@ -78,6 +80,11 @@ public class TextField extends FieldType
return parseFieldQuery(parser, getQueryAnalyzer(), field.getName(), externalVal);
}
+ @Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return ByteUtils.UTF8toUTF16(term);
+ }
+
static Query parseFieldQuery(QParser parser, Analyzer analyzer, String field, String queryText) {
int phraseSlop = 0;
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieDateField.java Fri Nov 12 09:15:30 2010
@@ -73,6 +73,11 @@ public class TrieDateField extends DateF
}
@Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ return new Date(NumericUtils.prefixCodedToLong(term));
+ }
+
+ @Override
public SortField getSortField(SchemaField field, boolean top) {
return new SortField(new LongValuesCreator( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, CachedArrayCreator.CACHE_VALUES_AND_BITS ), top);
}
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieField.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/TrieField.java Fri Nov 12 09:15:30 2010
@@ -480,6 +480,24 @@ public class TrieField extends FieldType
}
@Override
+ public Object toObject(SchemaField sf, BytesRef term) {
+ switch (type) {
+ case INTEGER:
+ return NumericUtils.prefixCodedToInt(term);
+ case FLOAT:
+ return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term));
+ case LONG:
+ return NumericUtils.prefixCodedToLong(term);
+ case DOUBLE:
+ return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term));
+ case DATE:
+ return new Date(NumericUtils.prefixCodedToLong(term));
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
+ }
+ }
+
+ @Override
public String storedToIndexed(Fieldable f) {
// TODO: optimize to remove redundant string conversion
return readableToIndexed(storedToReadable(f));
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/Grouping.java Fri Nov 12 09:15:30 2010
@@ -19,6 +19,8 @@ package org.apache.solr.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.*;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.ValueSource;
@@ -27,25 +29,287 @@ import java.util.*;
public class Grouping {
- public static class Command {
- public String key; // the name to use for this group in the response
- public Sort groupSort; // the sort of the documents *within* a single group.
+
+ public abstract class Command {
+ public String key; // the name to use for this group in the response
+ public Sort groupSort; // the sort of the documents *within* a single group.
+ public Sort sort; // the sort between groups
public int docsPerGroup; // how many docs in each group - from "group.limit" param, default=1
- public int numGroups; // how many groups - defaults to the "rows" parameter
+ public int groupOffset; // the offset within each group (for paging within each group)
+ public int numGroups; // how many groups - defaults to the "rows" parameter
+ public int offset; // offset into the list of groups
+
+
+ abstract void prepare() throws IOException;
+ abstract Collector createCollector() throws IOException;
+ Collector createNextCollector() throws IOException {
+ return null;
+ }
+ abstract void finish() throws IOException;
+
+ abstract int getMatches();
+
+ NamedList commonResponse() {
+ NamedList groupResult = new SimpleOrderedMap();
+ grouped.add(key, groupResult); // grouped={ key={
+
+ int this_matches = getMatches();
+ groupResult.add("matches", this_matches);
+ maxMatches = Math.max(maxMatches, this_matches);
+ return groupResult;
+ }
+
+ DocList getDocList(TopDocsCollector collector) {
+ int docsToCollect = getMax(groupOffset, docsPerGroup, maxDoc);
+
+ // TODO: implement a DocList impl that doesn't need to start at offset=0
+ TopDocs topDocs = collector.topDocs(0, docsToCollect);
+
+ int ids[] = new int[topDocs.scoreDocs.length];
+ float[] scores = needScores ? new float[topDocs.scoreDocs.length] : null;
+ for (int i=0; i<ids.length; i++) {
+ ids[i] = topDocs.scoreDocs[i].doc;
+ if (scores != null)
+ scores[i] = topDocs.scoreDocs[i].score;
+ }
+
+ float score = topDocs.getMaxScore();
+ maxScore = Math.max(maxScore, score);
+ DocSlice docs = new DocSlice(groupOffset, Math.max(0, ids.length - groupOffset), ids, scores, topDocs.totalHits, score);
+
+ if (getDocList) {
+ DocIterator iter = docs.iterator();
+ while (iter.hasNext())
+ idSet.add(iter.nextDoc());
+ }
+ return docs;
+ }
+
+ void addDocList(NamedList rsp, TopDocsCollector collector) {
+ rsp.add("doclist", getDocList(collector));
+ }
}
- public static class CommandQuery extends Command {
+ public class CommandQuery extends Command {
public Query query;
+
+ TopDocsCollector topCollector;
+ FilterCollector collector;
+
+ @Override
+ void prepare() throws IOException {
+ }
+
+ @Override
+ Collector createCollector() throws IOException {
+ int docsToCollect = getMax(groupOffset, docsPerGroup, maxDoc);
+ DocSet groupFilt = searcher.getDocSet(query);
+ topCollector = newCollector(groupSort, docsToCollect, false, needScores);
+ collector = new FilterCollector(groupFilt, topCollector);
+ return collector;
+ }
+
+ @Override
+ void finish() throws IOException {
+ NamedList rsp = commonResponse();
+ addDocList(rsp, (TopDocsCollector)collector.getCollector());
+ }
+
+ @Override
+ int getMatches() {
+ return collector.getMatches();
+ }
}
- public static class CommandFunc extends Command {
+
+ public class CommandFunc extends Command {
public ValueSource groupBy;
- // todo - find a better place to store these
- transient Map context;
- transient Collector collector;
+ int maxGroupToFind;
+ Map context;
+ TopGroupCollector collector = null;
+ Phase2GroupCollector collector2;
+
+ @Override
+ void prepare() throws IOException {
+ Map context = ValueSource.newContext();
+ groupBy.createWeight(context, searcher);
+ }
+
+ @Override
+ Collector createCollector() throws IOException {
+ maxGroupToFind = getMax(offset, numGroups, maxDoc);
+
+ if (compareSorts(sort, groupSort)) {
+ collector = new TopGroupCollector(groupBy, context, normalizeSort(sort), maxGroupToFind);
+ } else {
+ collector = new TopGroupSortCollector(groupBy, context, normalizeSort(sort), normalizeSort(groupSort), maxGroupToFind);
+ }
+ return collector;
+ }
+
+ @Override
+ Collector createNextCollector() throws IOException {
+ int docsToCollect = getMax(groupOffset, docsPerGroup, maxDoc);
+ if (docsToCollect < 0 || docsToCollect > maxDoc) docsToCollect = maxDoc;
+
+ collector2 = new Phase2GroupCollector(collector, groupBy, context, groupSort, docsToCollect, needScores, offset);
+ return collector2;
+ }
+
+ @Override
+ void finish() throws IOException {
+ NamedList groupResult = commonResponse();
+
+ if (collector.orderedGroups == null) collector.buildSet();
+
+ List groupList = new ArrayList();
+ groupResult.add("groups", groupList); // grouped={ key={ groups=[
+
+ int skipCount = offset;
+ for (SearchGroup group : collector.orderedGroups) {
+ if (skipCount > 0) {
+ skipCount--;
+ continue;
+ }
+ NamedList nl = new SimpleOrderedMap();
+ groupList.add(nl); // grouped={ key={ groups=[ {
+
+ nl.add("groupValue", group.groupValue.toObject());
+
+ SearchGroupDocs groupDocs = collector2.groupMap.get(group.groupValue);
+ addDocList(nl, groupDocs.collector);
+ }
+ }
+
+ @Override
+ int getMatches() {
+ return collector.getMatches();
+ }
+ }
+
+
+
+ static Sort byScoreDesc = new Sort();
+
+ static boolean compareSorts(Sort sort1, Sort sort2) {
+ return sort1 == sort2 || normalizeSort(sort1).equals(normalizeSort(sort2));
+ }
+
+ /** returns a sort by score desc if null */
+ static Sort normalizeSort(Sort sort) {
+ return sort==null ? byScoreDesc : sort;
+ }
+
+ static int getMax(int offset, int len, int max) {
+ int v = len<0 ? max : offset + len;
+ if (v < 0 || v > max) v = max;
+ return v;
}
+
+ static TopDocsCollector newCollector(Sort sort, int numHits, boolean fillFields, boolean needScores) throws IOException {
+ if (sort==null || sort==byScoreDesc) {
+ return TopScoreDocCollector.create(numHits, true);
+ } else {
+ return TopFieldCollector.create(sort, numHits, false, needScores, needScores, true);
+ }
+ }
+
+
+ final SolrIndexSearcher searcher;
+ final SolrIndexSearcher.QueryResult qr;
+ final SolrIndexSearcher.QueryCommand cmd;
+ final List<Command> commands = new ArrayList<Command>();
+
+ public Grouping(SolrIndexSearcher searcher, SolrIndexSearcher.QueryResult qr, SolrIndexSearcher.QueryCommand cmd) {
+ this.searcher = searcher;
+ this.qr = qr;
+ this.cmd = cmd;
+ }
+
+ public void add(Grouping.Command groupingCommand) {
+ commands.add(groupingCommand);
+ }
+
+ int maxDoc;
+ boolean needScores;
+ boolean getDocSet;
+ boolean getDocList; // doclist needed for debugging or highlighting
+ Query query;
+ DocSet filter;
+ Filter luceneFilter;
+ NamedList grouped = new SimpleOrderedMap();
+ Set<Integer> idSet = new LinkedHashSet<Integer>(); // used for tracking unique docs when we need a doclist
+ int maxMatches; // max number of matches from any grouping command
+ float maxScore = Float.NEGATIVE_INFINITY; // max score seen in any doclist
+
+ public void execute() throws IOException {
+ DocListAndSet out = new DocListAndSet();
+ qr.setDocListAndSet(out);
+
+ filter = cmd.getFilter()!=null ? cmd.getFilter() : searcher.getDocSet(cmd.getFilterList());
+ luceneFilter = filter == null ? null : filter.getTopFilter();
+
+ maxDoc = searcher.maxDoc();
+
+ needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
+ getDocSet = (cmd.getFlags() & SolrIndexSearcher.GET_DOCSET) != 0;
+ getDocList = (cmd.getFlags() & SolrIndexSearcher.GET_DOCLIST) != 0; // doclist needed for debugging or highlighting
+ query = QueryUtils.makeQueryable(cmd.getQuery());
+
+ for (Command cmd : commands) {
+ cmd.prepare();
+ }
+
+ List<Collector> collectors = new ArrayList<Collector>(commands.size());
+ for (Command cmd : commands) {
+ Collector collector = cmd.createCollector();
+ if (collector != null)
+ collectors.add(collector);
+ }
+
+ Collector allCollectors = MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()]));
+ DocSetCollector setCollector = null;
+ if (getDocSet) {
+ setCollector = new DocSetDelegateCollector(maxDoc>>6, maxDoc, allCollectors);
+ allCollectors = setCollector;
+ }
+
+ searcher.search(query, luceneFilter, allCollectors);
+
+ if (getDocSet) {
+ qr.setDocSet(setCollector.getDocSet());
+ }
+
+ collectors.clear();
+ for (Command cmd : commands) {
+ Collector collector = cmd.createNextCollector();
+ if (collector != null)
+ collectors.add(collector);
+ }
+
+ if (collectors.size() > 0) {
+ searcher.search(query, luceneFilter, MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()])));
+ }
+
+ for (Command cmd : commands) {
+ cmd.finish();
+ }
+
+ qr.groupedResults = grouped;
+
+ if (getDocList) {
+ int sz = idSet.size();
+ int[] ids = new int[sz];
+ int idx = 0;
+ for (int val : idSet) {
+ ids[idx++] = val;
+ }
+ qr.setDocList(new DocSlice(0, sz, ids, null, maxMatches, maxScore));
+ }
+ }
+
}
@@ -80,11 +344,11 @@ abstract class GroupCollector extends Co
class FilterCollector extends GroupCollector {
private final DocSet filter;
- private final TopFieldCollector collector;
+ private final Collector collector;
private int docBase;
private int matches;
- public FilterCollector(DocSet filter, TopFieldCollector collector) throws IOException {
+ public FilterCollector(DocSet filter, Collector collector) throws IOException {
this.filter = filter;
this.collector = collector;
}
@@ -117,7 +381,7 @@ class FilterCollector extends GroupColle
return matches;
}
- TopFieldCollector getTopFieldCollector() {
+ Collector getCollector() {
return collector;
}
}
@@ -511,14 +775,24 @@ class Phase2GroupCollector extends Colle
int docBase;
// TODO: may want to decouple from the phase1 collector
- public Phase2GroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext, Sort sort, int docsPerGroup, boolean getScores) throws IOException {
+ public Phase2GroupCollector(TopGroupCollector topGroups, ValueSource groupByVS, Map vsContext, Sort sort, int docsPerGroup, boolean getScores, int offset) throws IOException {
boolean getSortFields = false;
+ if (topGroups.orderedGroups == null)
+ topGroups.buildSet();
+
groupMap = new HashMap<MutableValue, SearchGroupDocs>(topGroups.groupMap.size());
- for (SearchGroup group : topGroups.groupMap.values()) {
+ for (SearchGroup group : topGroups.orderedGroups) {
+ if (offset > 0) {
+ offset--;
+ continue;
+ }
SearchGroupDocs groupDocs = new SearchGroupDocs();
groupDocs.groupValue = group.groupValue;
- groupDocs.collector = TopFieldCollector.create(sort, docsPerGroup, getSortFields, getScores, getScores, true);
+ if (sort==null)
+ groupDocs.collector = TopScoreDocCollector.create(docsPerGroup, true);
+ else
+ groupDocs.collector = TopFieldCollector.create(sort, docsPerGroup, getSortFields, getScores, getScores, true);
groupMap.put(groupDocs.groupValue, groupDocs);
}
@@ -562,6 +836,6 @@ class Phase2GroupCollector extends Colle
// disad: blows up the size of SearchGroup if we need many of them, and couples implementations
class SearchGroupDocs {
public MutableValue groupValue;
- TopFieldCollector collector;
+ TopDocsCollector collector;
}
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Fri Nov 12 09:15:30 2010
@@ -983,215 +983,19 @@ public class SolrIndexSearcher extends I
return qr.getDocList();
}
- private static final int NO_CHECK_QCACHE = 0x80000000;
- private static final int GET_DOCSET = 0x40000000;
- private static final int NO_CHECK_FILTERCACHE = 0x20000000;
+ static final int NO_CHECK_QCACHE = 0x80000000;
+ static final int GET_DOCSET = 0x40000000;
+ static final int NO_CHECK_FILTERCACHE = 0x20000000;
public static final int GET_DOCLIST = 0x02; // get the documents actually returned in a response
public static final int GET_SCORES = 0x01;
- private void groupBy(QueryResult qr, QueryCommand cmd) throws IOException {
- DocListAndSet out = new DocListAndSet();
- qr.setDocListAndSet(out);
-
- DocSet filter = cmd.getFilter()!=null ? cmd.getFilter() : getDocSet(cmd.getFilterList());
-
- int last = cmd.getOffset() + cmd.getLen();
- if (last < 0 || last > maxDoc()) last=maxDoc();
-
- boolean needScores = (cmd.getFlags() & GET_SCORES) != 0;
- boolean getDocSet = (cmd.getFlags() & GET_DOCSET) != 0;
- boolean getDocList = (cmd.getFlags() & GET_DOCLIST) != 0; // doclist needed for debugging or highlighting
- Query query = QueryUtils.makeQueryable(cmd.getQuery());
-
- final Filter luceneFilter = filter==null ? null : filter.getTopFilter();
-
- Sort sort = cmd.getSort();
- if (sort == null) sort = new Sort();
-
- List<GroupCollector> collectors = new ArrayList<GroupCollector>(cmd.groupCommands.size());
- for (Grouping.Command groupCommand : cmd.groupCommands) {
- // TODO: perhaps use some methods rather than instanceof
- if (groupCommand instanceof Grouping.CommandFunc) {
- Grouping.CommandFunc gc = (Grouping.CommandFunc)groupCommand;
- Map context = ValueSource.newContext();
- gc.groupBy.createWeight(context, this);
- TopGroupCollector collector;
- if (gc.groupSort != null && gc.groupSort != sort) {
- collector = new TopGroupSortCollector(gc.groupBy, context, sort, gc.groupSort, last);
- } else {
- collector = new TopGroupCollector(gc.groupBy, context, sort, last);
- }
- collectors.add(collector);
-
- // for next phase
- gc.context = context;
- gc.collector = collector;
- }
-
- if (groupCommand instanceof Grouping.CommandQuery) {
- DocSet groupFilt = getDocSet(((Grouping.CommandQuery)groupCommand).query);
- TopFieldCollector collector = TopFieldCollector.create(groupCommand.groupSort==null ? sort : groupCommand.groupSort, groupCommand.docsPerGroup, false, needScores, needScores, true);
- collectors.add(new FilterCollector(groupFilt, collector));
- }
- }
-
- Collector allCollectors = MultiCollector.wrap(collectors.toArray(new Collector[collectors.size()]));
- DocSetCollector setCollector = null;
- if (getDocSet) {
- // TODO: can callCollectors be zero length?
- setCollector = new DocSetDelegateCollector(maxDoc()>>6, maxDoc(), allCollectors);
- allCollectors = setCollector;
- }
-
- search(query, luceneFilter, allCollectors);
-
- if (getDocSet) {
- qr.docListAndSet.docSet = setCollector.getDocSet();
- }
-
- // TODO: make this a generic collector list
- int numPhase2 = 0;
- List<Phase2GroupCollector> phase2Collectors = new ArrayList<Phase2GroupCollector>(cmd.groupCommands.size());
- for (Grouping.Command groupCommand : cmd.groupCommands) {
- if (groupCommand instanceof Grouping.CommandFunc) {
- Grouping.CommandFunc gc = (Grouping.CommandFunc)groupCommand;
- Sort collectorSort = gc.groupSort == null ? sort : gc.groupSort;
- Phase2GroupCollector collector = new Phase2GroupCollector((TopGroupCollector)gc.collector, gc.groupBy, gc.context, collectorSort, gc.docsPerGroup, needScores);
- phase2Collectors.add(collector);
- numPhase2++;
- } else if (groupCommand instanceof Grouping.CommandQuery) {
- phase2Collectors.add(null);
- } else {
- phase2Collectors.add(null);
- }
- }
-
- // TODO: optionally cache docs and feed them back through rather than re-searching
- if (numPhase2 > 0)
- search(query, luceneFilter, MultiCollector.wrap(phase2Collectors.toArray(new Collector[phase2Collectors.size()])));
-
- Set<Integer> idSet = new LinkedHashSet<Integer>(); // used for tracking unique docs when we need a doclist
- int maxMatches = 0;
- float maxScore = Float.NEGATIVE_INFINITY;
-
- NamedList grouped = new SimpleOrderedMap();
- for (int cmdnum=0; cmdnum<cmd.groupCommands.size(); cmdnum++) {
- Grouping.Command groupCommand = cmd.groupCommands.get(cmdnum);
- GroupCollector gcollector = collectors.get(cmdnum);
-
- NamedList groupResult = new SimpleOrderedMap();
- grouped.add(groupCommand.key, groupResult); // grouped={ key={
-
- int this_matches = gcollector.getMatches();
- groupResult.add("matches", this_matches);
- maxMatches = Math.max(maxMatches, this_matches);
-
- // TODO: refactor this
- if (groupCommand instanceof Grouping.CommandQuery) {
- TopDocs topDocs = ((FilterCollector)gcollector).getTopFieldCollector().topDocs(0, groupCommand.docsPerGroup);
-
- // TODO: refactor
-
- //topDocs.totalHits
- int ids[] = new int[topDocs.scoreDocs.length];
- float[] scores = needScores ? new float[topDocs.scoreDocs.length] : null;
- for (int i=0; i<ids.length; i++) {
- ids[i] = topDocs.scoreDocs[i].doc;
- if (scores != null)
- scores[i] = topDocs.scoreDocs[i].score;
- }
-
- float score = topDocs.getMaxScore();
- maxScore = Math.max(maxScore, score);
- DocSlice docs = new DocSlice(0, ids.length, ids, scores, topDocs.totalHits, score);
- groupResult.add("doclist", docs);
-
- if (getDocList) {
- for (int id : ids)
- idSet.add(id);
- }
-
- continue;
- }
-
- Grouping.CommandFunc groupCommandFunc = (Grouping.CommandFunc)groupCommand;
- TopGroupCollector collector = (TopGroupCollector)gcollector;
- Phase2GroupCollector collector2 = phase2Collectors.get(cmdnum);
-
- if (collector.orderedGroups == null) collector.buildSet();
-
- List groupList = new ArrayList();
- groupResult.add("groups", groupList); // grouped={ key={ groups=[
-
- for (SearchGroup group : collector.orderedGroups) {
- NamedList nl = new SimpleOrderedMap();
- groupList.add(nl); // grouped={ key={ groups=[ {
-
- nl.add("groupValue", group.groupValue.toObject());
-
- SearchGroupDocs groupDocs = collector2.groupMap.get(group.groupValue);
- // nl.add("matches", groupDocs.matches); // redundant with doclist.numFound from the doc list
-
- TopDocs topDocs = groupDocs.collector.topDocs(0, groupCommandFunc.docsPerGroup);
- //topDocs.totalHits
- int ids[] = new int[topDocs.scoreDocs.length];
- float[] scores = needScores ? new float[topDocs.scoreDocs.length] : null;
- for (int i=0; i<ids.length; i++) {
- ids[i] = topDocs.scoreDocs[i].doc;
- if (scores != null)
- scores[i] = topDocs.scoreDocs[i].score;
- }
-
- float score = topDocs.getMaxScore();
- maxScore = Math.max(maxScore, score);
- DocSlice docs = new DocSlice(0, ids.length, ids, scores, topDocs.totalHits, score);
- nl.add("doclist", docs);
-
- if (getDocList) {
- for (int id : ids)
- idSet.add(id);
- }
-
- /*** values from stage 1
- DocSlice docs = new DocSlice(0, 1, new int[] {group.topDoc}, null, 1, 0);
- nl.add("docs", docs);
-
- Object[] vals = new Object[collector.comparators.length];
- for (int i=0; i<vals.length; i++) {
- vals[i] = collector.comparators[i].value(group.comparatorSlot);
- }
- nl.add("groupSortValue", vals);
- groupResult.add(nl);
- ***/
- }
- }
-
- qr.groupedResults = grouped;
-
- if (getDocList) {
- int sz = idSet.size();
- int[] ids = new int[sz];
- int idx = 0;
- for (int val : idSet) {
- ids[idx++] = val;
- }
- qr.docListAndSet.docList = new DocSlice(0, sz, ids, null, maxMatches, maxScore);
- }
-
- }
-
/**
* getDocList version that uses+populates query and filter caches.
* In the event of a timeout, the cache is not populated.
*/
private void getDocListC(QueryResult qr, QueryCommand cmd) throws IOException {
- if (cmd.groupCommands != null) {
- groupBy(qr, cmd);
- return;
- }
-
DocListAndSet out = new DocListAndSet();
qr.setDocListAndSet(out);
QueryResultKey key=null;
@@ -2002,7 +1806,7 @@ public class SolrIndexSearcher extends I
private int flags;
private long timeAllowed = -1;
- public List<Grouping.Command> groupCommands;
+ // public List<Grouping.Command> groupCommands;
public Query getQuery() { return query; }
public QueryCommand setQuery(Query query) {
Modified: lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrQueryParser.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/SolrQueryParser.java Fri Nov 12 09:15:30 2010
@@ -30,6 +30,7 @@ import org.apache.lucene.util.Version;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.SpecialOperations;
import org.apache.lucene.analysis.Analyzer;
import org.apache.solr.analysis.*;
import org.apache.solr.common.SolrException;
@@ -202,37 +203,36 @@ public class SolrQueryParser extends Que
String type = schema.getFieldType(field).getTypeName();
ReversedWildcardFilterFactory factory = leadingWildcards.get(type);
if (factory != null) {
+ Term term = new Term(field, termStr);
+ // fsa representing the query
+ Automaton automaton = WildcardQuery.toAutomaton(term);
+ // TODO: we should likely use the automaton to calculate shouldReverse, too.
if (factory.shouldReverse(termStr)) {
- int len = termStr.length();
- char[] chars = new char[len+1];
- chars[0] = factory.getMarkerChar();
- termStr.getChars(0, len, chars, 1);
- ReversedWildcardFilter.reverse(chars, 1, len);
- termStr = new String(chars);
+ automaton = BasicOperations.concatenate(automaton, BasicAutomata.makeChar(factory.getMarkerChar()));
+ SpecialOperations.reverse(automaton);
} else {
// reverse wildcardfilter is active: remove false positives
- Term term = new Term(field, termStr);
- // fsa representing the query
- Automaton a = WildcardQuery.toAutomaton(term);
// fsa representing false positives (markerChar*)
Automaton falsePositives = BasicOperations.concatenate(
BasicAutomata.makeChar(factory.getMarkerChar()),
BasicAutomata.makeAnyString());
- return new AutomatonQuery(term, BasicOperations.minus(a, falsePositives)) {
- // override toString so its completely transparent
- @Override
- public String toString(String field) {
- StringBuilder buffer = new StringBuilder();
- if (!getField().equals(field)) {
- buffer.append(getField());
- buffer.append(":");
- }
- buffer.append(term.text());
- buffer.append(ToStringUtils.boost(getBoost()));
- return buffer.toString();
- }
- };
+ // subtract these away
+ automaton = BasicOperations.minus(automaton, falsePositives);
}
+ return new AutomatonQuery(term, automaton) {
+ // override toString so its completely transparent
+ @Override
+ public String toString(String field) {
+ StringBuilder buffer = new StringBuilder();
+ if (!getField().equals(field)) {
+ buffer.append(getField());
+ buffer.append(":");
+ }
+ buffer.append(term.text());
+ buffer.append(ToStringUtils.boost(getBoost()));
+ return buffer.toString();
+ }
+ };
}
Query q = super.getWildcardQuery(field, termStr);
if (q instanceof WildcardQuery) {
Propchange: lucene/dev/branches/docvalues/solr/src/maven/solr-core-pom.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template:949730,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/solr/src/maven/solr-core-pom.xml.template:949730,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/solr/src/maven/solr-core-pom.xml.template:967125-979432
-/lucene/dev/trunk/solr/src/maven/solr-core-pom.xml.template:1021634-1029001
+/lucene/dev/trunk/solr/src/maven/solr-core-pom.xml.template:1021634-1034284
/lucene/solr/branches/newtrunk/solr/src/maven/solr-core-pom.xml.template:924462
/lucene/solr/trunk/src/maven/solr-core-pom.xml.template:922950-923910,923912-925091
Propchange: lucene/dev/branches/docvalues/solr/src/maven/solr-solrj-pom.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template:949730,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/solr/src/maven/solr-solrj-pom.xml.template:949730,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/solr/src/maven/solr-solrj-pom.xml.template:967125-979432
-/lucene/dev/trunk/solr/src/maven/solr-solrj-pom.xml.template:1021634-1029001
+/lucene/dev/trunk/solr/src/maven/solr-solrj-pom.xml.template:1021634-1034284
/lucene/solr/branches/newtrunk/solr/src/maven/solr-solrj-pom.xml.template:924462
/lucene/solr/trunk/src/maven/solr-solrj-pom.xml.template:922950-923910,923912-925091
Modified: lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/images/solr.jpg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/images/solr.jpg?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/images/solr_FC.eps
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/images/solr_FC.eps?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/tutorial.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/tutorial.xml?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/tutorial.xml (original)
+++ lucene/dev/branches/docvalues/solr/src/site/src/documentation/content/xdocs/tutorial.xml Fri Nov 12 09:15:30 2010
@@ -45,14 +45,14 @@ To follow along with this tutorial, you
</p>
<ol>
- <li>Java 1.5 or greater. Some places you can get it are from
+ <li>Java 1.6 or greater. Some places you can get it are from
<a href="http://java.sun.com/j2se/downloads.html">OpenJDK</a>,
<a href="http://java.sun.com/j2se/downloads.html">Sun</a>,
<a href="http://www.ibm.com/developerworks/java/jdk/">IBM</a>, or
<a href="http://www.oracle.com/technology/products/jrockit/index.html">Oracle</a>.
<br/>
Running <code>java -version</code> at the command line should indicate a version
- number starting with 1.5. Gnu's GCJ is not supported and does not work with Solr.
+ number starting with 1.6. Gnu's GCJ is not supported and does not work with Solr.
</li>
<li>A <a href="http://www.apache.org/dyn/closer.cgi/lucene/solr/">Solr release</a>.
</li>
Propchange: lucene/dev/branches/docvalues/solr/src/solrj/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/solr/src/solrj/org:949730,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/solr/src/solrj/org:949730,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
/lucene/dev/branches/preflexfixes/solr/src/solrj/org:967125-979432
-/lucene/dev/trunk/solr/src/solrj/org:1021634-1029001
+/lucene/dev/trunk/solr/src/solrj/org:1021634-1034284
/lucene/solr/branches/newtrunk/solr/src/solrj/org:924462
/lucene/solr/trunk/src/solrj/org:922950-923910,923912-925091
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/TestGroupingSearch.java Fri Nov 12 09:15:30 2010
@@ -105,27 +105,6 @@ public class TestGroupingSearch extends
);
}
- @Test
- public void testGroupingGroupSortingName() {
- assertU(add(doc("id", "1","name", "author1", "title", "a book title")));
- assertU(add(doc("id", "2","name", "author1", "title", "the title")));
- assertU(add(doc("id", "3","name", "author2", "title", "book title")));
- assertU(add(doc("id", "4","name", "author2", "title", "the title")));
- assertU(commit());
-
- assertQ(req("q","title:title", "group", "true", "group.field","name", "group.sort", "title asc")
- ,"*[count(//arr[@name='groups']/lst) = 2]"
- ,"//arr[@name='groups']/lst[1]/str[@name='groupValue'][.='author2']"
- // ,"//arr[@name='groups']/lst[1]/int[@name='matches'][.='2']"
- ,"//arr[@name='groups']/lst[1]/result[@numFound='2']"
- ,"//arr[@name='groups']/lst[1]/result/doc/*[@name='id'][.='3']"
-
- ,"//arr[@name='groups']/lst[2]/str[@name='groupValue'][.='author1']"
- // ,"//arr[@name='groups']/lst[2]/int[@name='matches'][.='2']"
- ,"//arr[@name='groups']/lst[2]/result[@numFound='2']"
- ,"//arr[@name='groups']/lst[2]/result/doc/*[@name='id'][.='1']"
- );
- }
@Test
public void testGroupingGroupSortingWeight() {
@@ -193,6 +172,13 @@ public class TestGroupingSearch extends
"]}}"
);
+ // test that filtering cuts down the result set
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "fq",f+":2")
+ ,"/grouped=={'"+f+"':{'matches':3,'groups':[" +
+ "{'groupValue':2,'doclist':{'numFound':3,'start':0,'docs':[{'id':'4'}]}}" +
+ "]}}"
+ );
+
// test limiting the number of groups returned
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2")
,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
@@ -201,6 +187,19 @@ public class TestGroupingSearch extends
"]}}"
);
+ // test offset into group list
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","1", "start","1")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "{'groupValue':3,'doclist':{'numFound':2,'start':0,'docs':[{'id':'3'}]}}" +
+ "]}}"
+ );
+
+ // test big offset into group list
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","1", "start","100")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "]}}"
+ );
+
// test increasing the docs per group returned
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2", "group.limit","3")
,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
@@ -209,6 +208,22 @@ public class TestGroupingSearch extends
"]}}"
);
+ // test offset into each group
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2", "group.limit","3", "group.offset","1")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "{'groupValue':1,'doclist':{'numFound':3,'start':1,'docs':[{'id':'10'},{'id':'5'}]}}," +
+ "{'groupValue':3,'doclist':{'numFound':2,'start':1,'docs':[{'id':'6'}]}}" +
+ "]}}"
+ );
+
+ // test big offset into each group
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id", "rows","2", "group.limit","3", "group.offset","10")
+ ,"/grouped=={'"+f+"':{'matches':10,'groups':[" +
+ "{'groupValue':1,'doclist':{'numFound':3,'start':10,'docs':[]}}," +
+ "{'groupValue':3,'doclist':{'numFound':2,'start':10,'docs':[]}}" +
+ "]}}"
+ );
+
// test adding in scores
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.field",f, "fl","id,score", "rows","2", "group.limit","2", "indent","off")
,"/grouped/"+f+"/groups==" +
@@ -257,6 +272,18 @@ public class TestGroupingSearch extends
"'doclist':{'numFound':4,'start':0,'docs':[{'id':'3'},{'id':'4'},{'id':'2'}]}}}"
);
+ // group.query and offset
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.query","id:[2 TO 5]", "fl","id", "group.limit","3", "group.offset","2")
+ ,"/grouped=={'id:[2 TO 5]':{'matches':10," +
+ "'doclist':{'numFound':4,'start':2,'docs':[{'id':'2'},{'id':'5'}]}}}"
+ );
+
+ // group.query and big offset
+ assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true", "group.query","id:[2 TO 5]", "fl","id", "group.limit","3", "group.offset","10")
+ ,"/grouped=={'id:[2 TO 5]':{'matches':10," +
+ "'doclist':{'numFound':4,'start':10,'docs':[]}}}"
+ );
+
// multiple at once
assertJQ(req("fq",filt, "q","{!func}"+f2, "group","true",
"group.query","id:[2 TO 5]",
Modified: lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java (original)
+++ lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java Fri Nov 12 09:15:30 2010
@@ -19,6 +19,7 @@ package org.apache.solr.analysis;
import java.io.IOException;
import java.io.StringReader;
+import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
@@ -26,8 +27,10 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.AutomatonQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.SpecialOperations;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrQueryParser;
@@ -51,6 +54,8 @@ public class TestReversedWildcardFilterF
public void setUp() throws Exception {
super.setUp();
schema = new IndexSchema(solrConfig, getSchemaFile(), null);
+ clearIndex();
+ assertU(commit());
}
@Test
@@ -105,7 +110,7 @@ public class TestReversedWildcardFilterF
}
@Test
- public void testQueryParsing() throws IOException, ParseException {
+ public void testQueryParsing() throws Exception {
SolrQueryParser parserOne = new SolrQueryParser(schema, "one");
assertTrue(parserOne.getAllowLeadingWildcard());
@@ -115,28 +120,53 @@ public class TestReversedWildcardFilterF
// XXX note: this should be false, but for now we return true for any field,
// XXX if at least one field uses the reversing
assertTrue(parserThree.getAllowLeadingWildcard());
- String text = "one +two *hree f*ur fiv* *si\uD834\uDD1Ex";
- String expectedOne = "one:one +one:two one:\u0001eerh* one:\u0001ru*f one:fiv* one:\u0001x\uD834\uDD1Eis*";
- String expectedTwo = "two:one +two:two two:\u0001eerh* two:\u0001ru*f two:fiv* two:\u0001x\uD834\uDD1Eis*";
- String expectedThree = "three:one +three:two three:*hree three:f*ur three:fiv* three:*si\uD834\uDD1Ex";
- Query q = parserOne.parse(text);
- assertEquals(expectedOne, q.toString());
- q = parserTwo.parse(text);
- assertEquals(expectedTwo, q.toString());
- q = parserThree.parse(text);
- assertEquals(expectedThree, q.toString());
+
+ // add some docs
+ assertU(adoc("id", "1", "one", "one"));
+ assertU(adoc("id", "2", "two", "two"));
+ assertU(adoc("id", "3", "three", "three"));
+ assertU(adoc("id", "4", "one", "four"));
+ assertU(adoc("id", "5", "two", "five"));
+ assertU(adoc("id", "6", "three", "si\uD834\uDD1Ex"));
+ assertU(commit());
+
+ assertQ("should have matched",
+ req("+id:1 +one:one"),
+ "//result[@numFound=1]");
+
+ assertQ("should have matched",
+ req("+id:4 +one:f*ur"),
+ "//result[@numFound=1]");
+
+ assertQ("should have matched",
+ req("+id:6 +three:*si\uD834\uDD1Ex"),
+ "//result[@numFound=1]");
+
// test conditional reversal
- String condText = "*hree t*ree th*ee thr*e ?hree t?ree th?ee th?*ee " +
- "short*token ver*longtoken";
- String expected = "two:\u0001eerh* two:\u0001eer*t two:\u0001ee*ht " +
- "two:thr*e " +
- "two:\u0001eerh? two:\u0001eer?t " +
- "two:th?ee " +
- "two:th?*ee " +
- "two:short*token " +
- "two:\u0001nekotgnol*rev";
- q = parserTwo.parse(condText);
- assertEquals(expected, q.toString());
+ assertTrue(wasReversed(parserTwo, "*hree"));
+ assertTrue(wasReversed(parserTwo, "t*ree"));
+ assertTrue(wasReversed(parserTwo, "th*ee"));
+ assertFalse(wasReversed(parserTwo, "thr*e"));
+ assertTrue(wasReversed(parserTwo, "?hree"));
+ assertTrue(wasReversed(parserTwo, "t?ree"));
+ assertFalse(wasReversed(parserTwo, "th?ee"));
+ assertFalse(wasReversed(parserTwo, "th?*ee"));
+ assertFalse(wasReversed(parserTwo, "short*token"));
+ assertTrue(wasReversed(parserTwo, "ver*longtoken"));
+ }
+
+ /** fragile assert: depends on our implementation, but cleanest way to check for now */
+ private boolean wasReversed(SolrQueryParser qp, String query) throws Exception {
+ Query q = qp.parse(query);
+ if (!(q instanceof AutomatonQuery))
+ return false;
+ // this is a hack to get the protected Automaton field in AutomatonQuery,
+ // may break in later lucene versions - we have no getter... for good reasons.
+ final Field automatonField = AutomatonQuery.class.getDeclaredField("automaton");
+ automatonField.setAccessible(true);
+ Automaton automaton = (Automaton) automatonField.get(q);
+ String prefix = SpecialOperations.getCommonPrefix(automaton);
+ return prefix.length() > 0 && prefix.charAt(0) == '\u0001';
}
@Test