You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2012/05/08 23:06:13 UTC
svn commit: r1335768 [1/3] - in /lucene/dev/trunk/solr:
contrib/clustering/src/test-files/clustering/solr/conf/
contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/
contrib/dataimporthandler/src/test-files/dih/solr/conf/ contrib/extract...
Author: ryan
Date: Tue May 8 21:06:10 2012
New Revision: 1335768
URL: http://svn.apache.org/viewvc?rev=1335768&view=rev
Log:
SOLR-2857: add UpdateRequestHandler that handles xml,csv,json, and javabin
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
- copied, changed from r1335654, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java (with props)
Removed:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltXMLLoader.java
Modified:
lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml
lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml
lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-enableplugin.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-functionquery.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master1-keepOneBackup.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master1.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master2.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master3.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-slave.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-solcoreproperties.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-tlog.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig_perf.xml
lucene/dev/trunk/solr/core/src/test-files/solr/crazy-path-to-config.xml
lucene/dev/trunk/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/BinaryUpdateRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CSVRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTestBase.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactoryTest.java
lucene/dev/trunk/solr/example/example-DIH/solr/db/conf/solrconfig.xml
lucene/dev/trunk/solr/example/example-DIH/solr/mail/conf/solrconfig.xml
lucene/dev/trunk/solr/example/example-DIH/solr/rss/conf/solrconfig.xml
lucene/dev/trunk/solr/example/example-DIH/solr/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/example/example-DIH/solr/tika/conf/solrconfig.xml
lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml
lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml
lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/ContentStreamUpdateRequest.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
lucene/dev/trunk/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml
lucene/dev/trunk/solr/solrj/src/test-files/solrj/solr/shared/conf/solrconfig.xml
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java
lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java
Modified: lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml Tue May 8 21:06:10 2012
@@ -361,10 +361,7 @@
To enable solr1.1 behavior, remove the /update handler or change its path
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
-
- <!-- CSV update handler, loaded on demand -->
- <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" />
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
<!--
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml Tue May 8 21:06:10 2012
@@ -296,7 +296,7 @@
"update.processor.class" is the class name for the UpdateRequestProcessor. It is initalized
only once. This can not be changed for each request.
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+ <requestHandler name="/update" class="solr.UpdateRequestHandler">
<!--
<str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
-->
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml Tue May 8 21:06:10 2012
@@ -300,7 +300,7 @@
"update.processor.class" is the class name for the UpdateRequestProcessor. It is initalized
only once. This can not be changed for each request.
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+ <requestHandler name="/update" class="solr.UpdateRequestHandler">
<!--
<str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
-->
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml Tue May 8 21:06:10 2012
@@ -298,7 +298,7 @@
"update.processor.class" is the class name for the UpdateRequestProcessor. It is initalized
only once. This can not be changed for each request.
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+ <requestHandler name="/update" class="solr.UpdateRequestHandler">
<!--
<str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
-->
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml Tue May 8 21:06:10 2012
@@ -296,7 +296,7 @@
"update.processor.class" is the class name for the UpdateRequestProcessor. It is initalized
only once. This can not be changed for each request.
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" >
<!--
<str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
-->
Modified: lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue May 8 21:06:10 2012
@@ -28,7 +28,7 @@ import org.apache.solr.common.params.Upd
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.handler.ContentStreamLoader;
+import org.apache.solr.handler.loader.ContentStreamLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
@@ -134,8 +134,8 @@ public class ExtractingDocumentLoader ex
* @throws java.io.IOException
*/
@Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws IOException {
- errHeader = "ExtractingDocumentLoader: " + stream.getSourceInfo();
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp,
+ ContentStream stream, UpdateRequestProcessor processor) throws Exception {
Parser parser = null;
String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);
if (streamType != null) {
@@ -240,6 +240,4 @@ public class ExtractingDocumentLoader ex
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stream type of " + streamType + " didn't match any known parsers. Please supply the " + ExtractingParams.STREAM_TYPE + " parameter.");
}
}
-
-
}
Modified: lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java Tue May 8 21:06:10 2012
@@ -26,7 +26,7 @@ import org.apache.solr.request.SolrQuery
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.solr.handler.ContentStreamHandlerBase;
-import org.apache.solr.handler.ContentStreamLoader;
+import org.apache.solr.handler.loader.ContentStreamLoader;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.mime.MimeTypeException;
import org.slf4j.Logger;
Modified: lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml Tue May 8 21:06:10 2012
@@ -194,10 +194,7 @@
</lst>
</requestHandler>
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
- <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy">
- <bool name="httpCaching">false</bool>
- </requestHandler>
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
<requestHandler name="/update/extract" class="org.apache.solr.handler.extraction.ExtractingRequestHandler"/>
Modified: lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Tue May 8 21:06:10 2012
@@ -301,7 +301,7 @@ public class ExtractingRequestHandlerTes
BufferingRequestProcessor p = new BufferingRequestProcessor(null);
ExtractingDocumentLoader loader = (ExtractingDocumentLoader) handler.newLoader(req, p);
- loader.load(req, rsp, new ContentStreamBase.FileStream(getFile("extraction/version_control.txt")));
+ loader.load(req, rsp, new ContentStreamBase.FileStream(getFile("extraction/version_control.txt")),p);
AddUpdateCommand add = p.addCommands.get(0);
assertEquals(200, add.commitWithin);
Modified: lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml (original)
+++ lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml Tue May 8 21:06:10 2012
@@ -55,7 +55,7 @@
</httpCaching>
</requestDispatcher>
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler">
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" >
<lst name="defaults">
<str name="update.chain">lang_id</str>
</lst>
Modified: lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml Tue May 8 21:06:10 2012
@@ -705,15 +705,13 @@
code. To enable solr1.1 behavior, remove the /update handler or
change its path
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler">
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" >
<lst name="defaults">
<str name="update.chain">uima</str>
</lst>
</requestHandler>
- <requestHandler name="/update/javabin" class="solr.BinaryUpdateRequestHandler" />
-
<!--
Analysis request handler. Since Solr 1.3. Use to return how a
document is analyzed. Useful for debugging and as a token server for
@@ -803,7 +801,7 @@
signatureField and Solr will maintain uniqueness based on that
anyway. You have to link the chain to an update handler above to use
it ie: <requestHandler name="/update
- "class="solr.XmlUpdateRequestHandler"> <lst name="defaults"> <str
+ "class="solr.UpdateRequestHandler"> <lst name="defaults"> <str
name="update.chain">dedupe</str> </lst> </requestHandler>
-->
Modified: lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml Tue May 8 21:06:10 2012
@@ -705,9 +705,7 @@
code. To enable solr1.1 behavior, remove the /update handler or
change its path
-->
- <requestHandler name="/update" class="solr.XmlUpdateRequestHandler"/>
-
- <requestHandler name="/update/javabin" class="solr.BinaryUpdateRequestHandler" />
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
<!--
Analysis request handler. Since Solr 1.3. Use to return how a
@@ -798,7 +796,7 @@
signatureField and Solr will maintain uniqueness based on that
anyway. You have to link the chain to an update handler above to use
it ie: <requestHandler name="/update
- "class="solr.XmlUpdateRequestHandler"> <lst name="defaults"> <str
+ "class="solr.UpdateRequestHandler"> <lst name="defaults"> <str
name="update.chain">dedupe</str> </lst> </requestHandler>
-->
Modified: lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Tue May 8 21:06:10 2012
@@ -31,7 +31,7 @@ import org.apache.solr.common.params.Upd
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.XmlUpdateRequestHandler;
+import org.apache.solr.handler.UpdateRequestHandler;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
@@ -192,7 +192,7 @@ public class UIMAUpdateRequestProcessorT
SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), (SolrParams) mmparams) {
};
- XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
+ UpdateRequestHandler handler = new UpdateRequestHandler();
handler.init(null);
ArrayList<ContentStream> streams = new ArrayList<ContentStream>(2);
streams.add(new ContentStreamBase.StringStream(doc));
Modified: lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml Tue May 8 21:06:10 2012
@@ -83,18 +83,8 @@
</lst>
</requestHandler>
-
- <requestHandler name="/update"
- class="solr.XmlUpdateRequestHandler">
- </requestHandler>
- <!-- Binary Update Request Handler
- http://wiki.apache.org/solr/javabin
- -->
- <requestHandler name="/update/javabin"
- class="solr.BinaryUpdateRequestHandler" />
- <requestHandler name="/update/json"
- class="solr.JsonUpdateRequestHandler"
- startup="lazy" />
+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />
+
<!--
<queryResponseWriter name="xml"
default="true"
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java Tue May 8 21:06:10 2012
@@ -17,128 +17,24 @@
package org.apache.solr.handler;
-import org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.FastInputStream;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.update.DeleteUpdateCommand;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
+import org.apache.solr.common.util.NamedList;
/**
* Update handler which uses the JavaBin format
*
- *
* @see org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec
* @see org.apache.solr.common.util.JavaBinCodec
- * @since solr 1.4
+ *
+ * use {@link UpdateRequestHandler}
*/
-public class BinaryUpdateRequestHandler extends ContentStreamHandlerBase {
-
+@Deprecated
+public class BinaryUpdateRequestHandler extends UpdateRequestHandler {
@Override
- protected ContentStreamLoader newLoader(SolrQueryRequest req, final UpdateRequestProcessor processor) {
- return new ContentStreamLoader() {
- @Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws Exception {
- InputStream is = null;
- try {
- is = stream.getStream();
- parseAndLoadDocs(req, rsp, is, processor);
- } finally {
- if(is != null) {
- is.close();
- }
- }
- }
- };
- }
-
- private void parseAndLoadDocs(final SolrQueryRequest req, SolrQueryResponse rsp, InputStream stream,
- final UpdateRequestProcessor processor) throws IOException {
- UpdateRequest update = null;
- JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler() {
- private AddUpdateCommand addCmd = null;
-
- public void update(SolrInputDocument document, UpdateRequest updateRequest) {
- if (document == null) {
- // Perhaps commit from the parameters
- try {
- RequestHandlerUtils.handleCommit(req, processor, updateRequest.getParams(), false);
- RequestHandlerUtils.handleRollback(req, processor, updateRequest.getParams(), false);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ERROR handling commit/rollback");
- }
- return;
- }
- if (addCmd == null) {
- addCmd = getAddCommand(req, updateRequest.getParams());
- }
- addCmd.solrDoc = document;
- try {
- processor.processAdd(addCmd);
- addCmd.clear();
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ERROR adding document " + document);
- }
- }
- };
- FastInputStream in = FastInputStream.wrap(stream);
- for (; ; ) {
- try {
- update = new JavaBinUpdateRequestCodec().unmarshal(in, handler);
- } catch (EOFException e) {
- break; // this is expected
- } catch (Exception e) {
- log.error("Exception while processing update request", e);
- break;
- }
- if (update.getDeleteById() != null || update.getDeleteQuery() != null) {
- delete(req, update, processor);
- }
- }
- }
-
- private AddUpdateCommand getAddCommand(SolrQueryRequest req, SolrParams params) {
- AddUpdateCommand addCmd = new AddUpdateCommand(req);
-
- addCmd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
- addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
- return addCmd;
- }
-
- private void delete(SolrQueryRequest req, UpdateRequest update, UpdateRequestProcessor processor) throws IOException {
- SolrParams params = update.getParams();
- DeleteUpdateCommand delcmd = new DeleteUpdateCommand(req);
- if(params != null) {
- delcmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
- }
-
- if(update.getDeleteById() != null) {
- for (String s : update.getDeleteById()) {
- delcmd.id = s;
- processor.processDelete(delcmd);
- }
- delcmd.id = null;
- }
-
- if(update.getDeleteQuery() != null) {
- for (String s : update.getDeleteQuery()) {
- delcmd.query = s;
- processor.processDelete(delcmd);
- }
- }
+ public void init(NamedList args) {
+ super.init(args);
+ setAssumeContentType("application/javabin");
+ log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue May 8 21:06:10 2012
@@ -17,40 +17,19 @@
package org.apache.solr.handler;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.update.*;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.internal.csv.CSVStrategy;
-import org.apache.solr.internal.csv.CSVParser;
-import org.apache.commons.io.IOUtils;
-
-import java.util.regex.Pattern;
-import java.util.List;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.io.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.solr.common.util.NamedList;
/**
- *
+ * use {@link UpdateRequestHandler}
*/
-
-public class CSVRequestHandler extends ContentStreamHandlerBase {
- public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
+@Deprecated
+public class CSVRequestHandler extends UpdateRequestHandler {
@Override
- protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- return new SingleThreadedCSVLoader(req, processor);
+ public void init(NamedList args) {
+ super.init(args);
+ setAssumeContentType("application/csv");
+ log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
}
//////////////////////// SolrInfoMBeans methods //////////////////////
@@ -66,376 +45,6 @@ public class CSVRequestHandler extends C
}
-abstract class CSVLoader extends ContentStreamLoader {
- public static final String SEPARATOR="separator";
- public static final String FIELDNAMES="fieldnames";
- public static final String HEADER="header";
- public static final String SKIP="skip";
- public static final String SKIPLINES="skipLines";
- public static final String MAP="map";
- public static final String TRIM="trim";
- public static final String EMPTY="keepEmpty";
- public static final String SPLIT="split";
- public static final String ENCAPSULATOR="encapsulator";
- public static final String ESCAPE="escape";
- public static final String OVERWRITE="overwrite";
- public static final String LITERALS_PREFIX = "literal.";
-
- private static Pattern colonSplit = Pattern.compile(":");
- private static Pattern commaSplit = Pattern.compile(",");
-
- public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
-
- final IndexSchema schema;
- final SolrParams params;
- final CSVStrategy strategy;
- final UpdateRequestProcessor processor;
-
- // hashmap to save any literal fields and their values
- HashMap <SchemaField, String> literals;
- String[] fieldnames;
- SchemaField[] fields;
- CSVLoader.FieldAdder[] adders;
-
- int skipLines; // number of lines to skip at start of file
-
- final AddUpdateCommand templateAdd;
-
-
-
- /** Add a field to a document unless it's zero length.
- * The FieldAdder hierarchy handles all the complexity of
- * further transforming or splitting field values to keep the
- * main logic loop clean. All implementations of add() must be
- * MT-safe!
- */
- private class FieldAdder {
- void add(SolrInputDocument doc, int line, int column, String val) {
- if (val.length() > 0) {
- doc.addField(fields[column].getName(),val,1.0f);
- }
- }
- }
-
- /** add zero length fields */
- private class FieldAdderEmpty extends CSVLoader.FieldAdder {
- @Override
- void add(SolrInputDocument doc, int line, int column, String val) {
- doc.addField(fields[column].getName(),val,1.0f);
- }
- }
-
- /** trim fields */
- private class FieldTrimmer extends CSVLoader.FieldAdder {
- private final CSVLoader.FieldAdder base;
- FieldTrimmer(CSVLoader.FieldAdder base) { this.base=base; }
- @Override
- void add(SolrInputDocument doc, int line, int column, String val) {
- base.add(doc, line, column, val.trim());
- }
- }
-
- /** map a single value.
- * for just a couple of mappings, this is probably faster than
- * using a HashMap.
- */
- private class FieldMapperSingle extends CSVLoader.FieldAdder {
- private final String from;
- private final String to;
- private final CSVLoader.FieldAdder base;
- FieldMapperSingle(String from, String to, CSVLoader.FieldAdder base) {
- this.from=from;
- this.to=to;
- this.base=base;
- }
- @Override
- void add(SolrInputDocument doc, int line, int column, String val) {
- if (from.equals(val)) val=to;
- base.add(doc,line,column,val);
- }
- }
-
- /** Split a single value into multiple values based on
- * a CSVStrategy.
- */
- private class FieldSplitter extends CSVLoader.FieldAdder {
- private final CSVStrategy strategy;
- private final CSVLoader.FieldAdder base;
- FieldSplitter(CSVStrategy strategy, CSVLoader.FieldAdder base) {
- this.strategy = strategy;
- this.base = base;
- }
-
- @Override
- void add(SolrInputDocument doc, int line, int column, String val) {
- CSVParser parser = new CSVParser(new StringReader(val), strategy);
- try {
- String[] vals = parser.getLine();
- if (vals!=null) {
- for (String v: vals) base.add(doc,line,column,v);
- } else {
- base.add(doc,line,column,val);
- }
- } catch (IOException e) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,e);
- }
- }
- }
-
-
- String errHeader="CSVLoader:";
-
- CSVLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- this.processor = processor;
- this.params = req.getParams();
- schema = req.getSchema();
- this.literals = new HashMap<SchemaField, String>();
-
- templateAdd = new AddUpdateCommand(req);
- templateAdd.overwrite=params.getBool(OVERWRITE,true);
- templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
-
- strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
- String sep = params.get(SEPARATOR);
- if (sep!=null) {
- if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
- strategy.setDelimiter(sep.charAt(0));
- }
-
- String encapsulator = params.get(ENCAPSULATOR);
- if (encapsulator!=null) {
- if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
- }
-
- String escape = params.get(ESCAPE);
- if (escape!=null) {
- if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
- }
-
- // if only encapsulator or escape is set, disable the other escaping mechanism
- if (encapsulator == null && escape != null) {
- strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
- strategy.setEscape(escape.charAt(0));
- } else {
- if (encapsulator != null) {
- strategy.setEncapsulator(encapsulator.charAt(0));
- }
- if (escape != null) {
- char ch = escape.charAt(0);
- strategy.setEscape(ch);
- if (ch == '\\') {
- // If the escape is the standard backslash, then also enable
- // unicode escapes (it's harmless since 'u' would not otherwise
- // be escaped.
- strategy.setUnicodeEscapeInterpretation(true);
- }
- }
- }
-
- String fn = params.get(FIELDNAMES);
- fieldnames = fn != null ? commaSplit.split(fn,-1) : null;
-
- Boolean hasHeader = params.getBool(HEADER);
-
- skipLines = params.getInt(SKIPLINES,0);
-
- if (fieldnames==null) {
- if (null == hasHeader) {
- // assume the file has the headers if they aren't supplied in the args
- hasHeader=true;
- } else if (!hasHeader) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"CSVLoader: must specify fieldnames=<fields>* or header=true");
- }
- } else {
- // if the fieldnames were supplied and the file has a header, we need to
- // skip over that header.
- if (hasHeader!=null && hasHeader) skipLines++;
-
- prepareFields();
- }
- }
-
- /** create the FieldAdders that control how each field is indexed */
- void prepareFields() {
- // Possible future optimization: for really rapid incremental indexing
- // from a POST, one could cache all of this setup info based on the params.
- // The link from FieldAdder to this would need to be severed for that to happen.
-
- fields = new SchemaField[fieldnames.length];
- adders = new CSVLoader.FieldAdder[fieldnames.length];
- String skipStr = params.get(SKIP);
- List<String> skipFields = skipStr==null ? null : StrUtils.splitSmart(skipStr,',');
-
- CSVLoader.FieldAdder adder = new CSVLoader.FieldAdder();
- CSVLoader.FieldAdder adderKeepEmpty = new CSVLoader.FieldAdderEmpty();
-
- for (int i=0; i<fields.length; i++) {
- String fname = fieldnames[i];
- // to skip a field, leave the entries in fields and addrs null
- if (fname.length()==0 || (skipFields!=null && skipFields.contains(fname))) continue;
-
- fields[i] = schema.getField(fname);
- boolean keepEmpty = params.getFieldBool(fname,EMPTY,false);
- adders[i] = keepEmpty ? adderKeepEmpty : adder;
-
- // Order that operations are applied: split -> trim -> map -> add
- // so create in reverse order.
- // Creation of FieldAdders could be optimized and shared among fields
-
- String[] fmap = params.getFieldParams(fname,MAP);
- if (fmap!=null) {
- for (String mapRule : fmap) {
- String[] mapArgs = colonSplit.split(mapRule,-1);
- if (mapArgs.length!=2)
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Map rules must be of the form 'from:to' ,got '"+mapRule+"'");
- adders[i] = new CSVLoader.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
- }
- }
-
- if (params.getFieldBool(fname,TRIM,false)) {
- adders[i] = new CSVLoader.FieldTrimmer(adders[i]);
- }
-
- if (params.getFieldBool(fname,SPLIT,false)) {
- String sepStr = params.getFieldParam(fname,SEPARATOR);
- char fsep = sepStr==null || sepStr.length()==0 ? ',' : sepStr.charAt(0);
- String encStr = params.getFieldParam(fname,ENCAPSULATOR);
- char fenc = encStr==null || encStr.length()==0 ? (char)-2 : encStr.charAt(0);
- String escStr = params.getFieldParam(fname,ESCAPE);
- char fesc = escStr==null || escStr.length()==0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0);
-
- CSVStrategy fstrat = new CSVStrategy(fsep,fenc,CSVStrategy.COMMENTS_DISABLED,fesc, false, false, false, false);
- adders[i] = new CSVLoader.FieldSplitter(fstrat, adders[i]);
- }
- }
- // look for any literal fields - literal.foo=xyzzy
- Iterator<String> paramNames = params.getParameterNamesIterator();
- while (paramNames.hasNext()) {
- String pname = paramNames.next();
- if (!pname.startsWith(LITERALS_PREFIX)) continue;
-
- String name = pname.substring(LITERALS_PREFIX.length());
- SchemaField sf = schema.getFieldOrNull(name);
- if(sf == null)
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
- literals.put(sf, params.get(pname));
- }
- }
-
- private void input_err(String msg, String[] line, int lineno) {
- StringBuilder sb = new StringBuilder();
- sb.append(errHeader).append(", line=").append(lineno).append(",").append(msg).append("\n\tvalues={");
- for (String val: line) {
- sb.append("'").append(val).append("',"); }
- sb.append('}');
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString());
- }
-
- private void input_err(String msg, String[] lines, int lineNo, Throwable e) {
- StringBuilder sb = new StringBuilder();
- sb.append(errHeader).append(", line=").append(lineNo).append(",").append(msg).append("\n\tvalues={");
- if (lines != null) {
- for (String val : lines) {
- sb.append("'").append(val).append("',");
- }
- } else {
- sb.append("NO LINES AVAILABLE");
- }
- sb.append('}');
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString(), e);
- }
-
- /** load the CSV input */
- @Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws IOException {
- errHeader = "CSVLoader: input=" + stream.getSourceInfo();
- Reader reader = null;
- try {
- reader = stream.getReader();
- if (skipLines>0) {
- if (!(reader instanceof BufferedReader)) {
- reader = new BufferedReader(reader);
- }
- BufferedReader r = (BufferedReader)reader;
- for (int i=0; i<skipLines; i++) {
- r.readLine();
- }
- }
-
- CSVParser parser = new CSVParser(reader, strategy);
-
- // parse the fieldnames from the header of the file
- if (fieldnames==null) {
- fieldnames = parser.getLine();
- if (fieldnames==null) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
- }
- prepareFields();
- }
-
- // read the rest of the CSV file
- for(;;) {
- int line = parser.getLineNumber(); // for error reporting in MT mode
- String[] vals = null;
- try {
- vals = parser.getLine();
- } catch (IOException e) {
- //Catch the exception and rethrow it with more line information
- input_err("can't read line: " + line, null, line, e);
- }
- if (vals==null) break;
-
- if (vals.length != fields.length) {
- input_err("expected "+fields.length+" values but got "+vals.length, vals, line);
- }
-
- addDoc(line,vals);
- }
- } finally{
- if (reader != null) {
- IOUtils.closeQuietly(reader);
- }
- }
- }
-
- /** called for each line of values (document) */
- abstract void addDoc(int line, String[] vals) throws IOException;
-
- /** this must be MT safe... may be called concurrently from multiple threads. */
- void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template) throws IOException {
- // the line number is passed simply for error reporting in MT mode.
- // first, create the lucene document
- for (int i=0; i<vals.length; i++) {
- if (fields[i]==null) continue; // ignore this field
- String val = vals[i];
- adders[i].add(doc, line, i, val);
- }
-
- // add any literals
- for (SchemaField sf : literals.keySet()) {
- String fn = sf.getName();
- String val = literals.get(sf);
- doc.addField(fn, val);
- }
-
- template.solrDoc = doc;
- processor.processAdd(template);
- }
-
-}
-
-
-class SingleThreadedCSVLoader extends CSVLoader {
- SingleThreadedCSVLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- super(req, processor);
- }
- @Override
- void addDoc(int line, String[] vals) throws IOException {
- templateAdd.clear();
- SolrInputDocument doc = new SolrInputDocument();
- doAdd(line, vals, doc, templateAdd);
- }
-}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java Tue May 8 21:06:10 2012
@@ -21,6 +21,7 @@ import org.apache.solr.common.params.Sol
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.loader.ContentStreamLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.processor.UpdateRequestProcessor;
@@ -30,9 +31,9 @@ import org.slf4j.LoggerFactory;
/**
- * Shares common code between various handlers that manipulate {@link org.apache.solr.common.util.ContentStream} objects.
- *
- **/
+ * Shares common code between various handlers that manipulate
+ * {@link org.apache.solr.common.util.ContentStream} objects.
+ */
public abstract class ContentStreamHandlerBase extends RequestHandlerBase {
public static Logger log = LoggerFactory.getLogger(ContentStreamHandlerBase.class);
@@ -70,7 +71,7 @@ public abstract class ContentStreamHandl
} else {
for (ContentStream stream : streams) {
- documentLoader.load(req, rsp, stream);
+ documentLoader.load(req, rsp, stream, processor);
}
// Perhaps commit from the parameters
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java Tue May 8 21:06:10 2012
@@ -18,26 +18,18 @@
package org.apache.solr.handler;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * Add documents to solr using the noggit JSON parser.
+ * use {@link UpdateRequestHandler}
*/
-public class JsonUpdateRequestHandler extends ContentStreamHandlerBase {
- public static Logger log = LoggerFactory.getLogger(JsonUpdateRequestHandler.class);
-
+@Deprecated
+public class JsonUpdateRequestHandler extends UpdateRequestHandler {
@Override
public void init(NamedList args) {
super.init(args);
- }
-
- @Override
- protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- return new JsonLoader(req, processor);
+ setAssumeContentType("application/json");
+ log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
}
//////////////////////// SolrInfoMBeans methods //////////////////////
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java Tue May 8 21:06:10 2012
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.loader.CSVLoader;
+import org.apache.solr.handler.loader.ContentStreamLoader;
+import org.apache.solr.handler.loader.JavabinLoader;
+import org.apache.solr.handler.loader.JsonLoader;
+import org.apache.solr.handler.loader.XMLLoader;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * UpdateHandler that uses content-type to pick the right Loader
+ */
+public class UpdateRequestHandler extends ContentStreamHandlerBase {
+ public static Logger log = LoggerFactory.getLogger(UpdateRequestHandler.class);
+
+ // XML Constants
+ public static final String ADD = "add";
+ public static final String DELETE = "delete";
+ public static final String OPTIMIZE = "optimize";
+ public static final String COMMIT = "commit";
+ public static final String ROLLBACK = "rollback";
+ public static final String WAIT_SEARCHER = "waitSearcher";
+ public static final String SOFT_COMMIT = "softCommit";
+
+ public static final String OVERWRITE = "overwrite";
+
+ public static final String VERSION = "version";
+
+ // NOTE: This constant is for use with the <add> XML tag, not the HTTP param with same name
+ public static final String COMMIT_WITHIN = "commitWithin";
+
+ Map<String,ContentStreamLoader> loaders = null;
+
+ ContentStreamLoader instance = new ContentStreamLoader() {
+ @Override
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp,
+ ContentStream stream, UpdateRequestProcessor processor) throws Exception {
+
+ String type = req.getParams().get(UpdateParams.ASSUME_CONTENT_TYPE);
+ if(type == null) {
+ type = stream.getContentType();
+ }
+ if( type == null ) { // Normal requests will not get here.
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Missing ContentType");
+ }
+ int idx = type.indexOf(';');
+ if(idx>0) {
+ type = type.substring(0,idx);
+ }
+ ContentStreamLoader loader = loaders.get(type);
+ if(loader==null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported ContentType: "
+ +type+ " Not in: "+loaders.keySet());
+ }
+ if(loader.getDefaultWT()!=null) {
+ setDefaultWT(req,loader);
+ }
+ loader.load(req, rsp, stream, processor);
+ }
+
+ private void setDefaultWT(SolrQueryRequest req, ContentStreamLoader loader) {
+ SolrParams params = req.getParams();
+ if( params.get(CommonParams.WT) == null ) {
+ String wt = loader.getDefaultWT();
+ // Make sure it is a valid writer
+ if(req.getCore().getQueryResponseWriter(wt)!=null) {
+ Map<String,String> map = new HashMap<String,String>(1);
+ map.put(CommonParams.WT, wt);
+ req.setParams(SolrParams.wrapDefaults(params,
+ new MapSolrParams(map)));
+ }
+ }
+ }
+ };
+
+ @Override
+ public void init(NamedList args) {
+ super.init(args);
+
+ // Since backed by a non-thread safe Map, it should not be modifiable
+ loaders = Collections.unmodifiableMap(createDefaultLoaders(args));
+ }
+
+ protected void setAssumeContentType(String ct) {
+ if(invariants==null) {
+ Map<String,String> map = new HashMap<String,String>();
+ map.put(UpdateParams.ASSUME_CONTENT_TYPE,ct);
+ invariants = new MapSolrParams(map);
+ }
+ else {
+ ModifiableSolrParams params = new ModifiableSolrParams(invariants);
+ params.set(UpdateParams.ASSUME_CONTENT_TYPE,ct);
+ invariants = params;
+ }
+ }
+
+ protected Map<String,ContentStreamLoader> createDefaultLoaders(NamedList args) {
+ SolrParams p = null;
+ if(args!=null) {
+ p = SolrParams.toSolrParams(args);
+ }
+ Map<String,ContentStreamLoader> registry = new HashMap<String,ContentStreamLoader>();
+ registry.put("application/xml", new XMLLoader().init(p) );
+ registry.put("application/json", new JsonLoader().init(p) );
+ registry.put("application/csv", new CSVLoader().init(p) );
+ registry.put("application/javabin", new JavabinLoader().init(p) );
+ registry.put("text/csv", registry.get("application/csv") );
+ registry.put("text/xml", registry.get("application/xml") );
+ registry.put("text/json", registry.get("application/json") );
+ return registry;
+ }
+
+
+ @Override
+ protected ContentStreamLoader newLoader(SolrQueryRequest req, final UpdateRequestProcessor processor) {
+ return instance;
+ }
+
+ //////////////////////// SolrInfoMBeans methods //////////////////////
+
+ @Override
+ public String getDescription() {
+ return "Add documents using XML (with XSLT), CSV, JSON, or javabin";
+ }
+
+ @Override
+ public String getSource() {
+ return "$URL$";
+ }
+}
+
+
+
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java Tue May 8 21:06:10 2012
@@ -17,67 +17,21 @@
package org.apache.solr.handler;
-import javax.xml.stream.XMLInputFactory;
-
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Add documents to solr using the STAX XML parser.
+ *
+ * use {@link UpdateRequestHandler}
*/
-public class XmlUpdateRequestHandler extends ContentStreamHandlerBase {
- public static Logger log = LoggerFactory.getLogger(XmlUpdateRequestHandler.class);
- private static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
-
- // XML Constants
- public static final String ADD = "add";
- public static final String DELETE = "delete";
- public static final String OPTIMIZE = "optimize";
- public static final String COMMIT = "commit";
- public static final String ROLLBACK = "rollback";
- public static final String WAIT_SEARCHER = "waitSearcher";
- public static final String SOFT_COMMIT = "softCommit";
-
- public static final String OVERWRITE = "overwrite";
-
- public static final String VERSION = "version";
-
- // NOTE: This constant is for use with the <add> XML tag, not the HTTP param with same name
- public static final String COMMIT_WITHIN = "commitWithin";
-
-
- XMLInputFactory inputFactory;
-
+@Deprecated
+public class XmlUpdateRequestHandler extends UpdateRequestHandler {
@Override
public void init(NamedList args) {
super.init(args);
-
- inputFactory = XMLInputFactory.newInstance();
- try {
- // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
- // XMLInputFactory, as that implementation tries to cache and reuse the
- // XMLStreamReader. Setting the parser-specific "reuse-instance" property to false
- // prevents this.
- // All other known open-source stax parsers (and the bea ref impl)
- // have thread-safe factories.
- inputFactory.setProperty("reuse-instance", Boolean.FALSE);
- }
- catch (IllegalArgumentException ex) {
- // Other implementations will likely throw this exception since "reuse-instance"
- // isimplementation specific.
- log.debug("Unable to set the 'reuse-instance' property for the input chain: " + inputFactory);
- }
- inputFactory.setXMLReporter(xmllog);
- }
-
- @Override
- protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- return new XMLLoader(processor, inputFactory);
+ setAssumeContentType("application/xml");
+ log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
}
//////////////////////// SolrInfoMBeans methods //////////////////////
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java Tue May 8 21:06:10 2012
@@ -17,58 +17,21 @@
package org.apache.solr.handler;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.xml.stream.XMLInputFactory;
/**
* Add documents to solr using the STAX XML parser, transforming it with XSLT first
+ *
+ * use {@link UpdateRequestHandler}
*/
-public class XsltUpdateRequestHandler extends ContentStreamHandlerBase {
- public static Logger log = LoggerFactory.getLogger(XsltUpdateRequestHandler.class);
- public static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
-
- public static final int XSLT_CACHE_DEFAULT = 60;
- private static final String XSLT_CACHE_PARAM = "xsltCacheLifetimeSeconds";
-
- XMLInputFactory inputFactory;
- private Integer xsltCacheLifetimeSeconds;
+@Deprecated
+public class XsltUpdateRequestHandler extends UpdateRequestHandler {
@Override
public void init(NamedList args) {
super.init(args);
-
- inputFactory = XMLInputFactory.newInstance();
- try {
- // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
- // XMLInputFactory, as that implementation tries to cache and reuse the
- // XMLStreamReader. Setting the parser-specific "reuse-instance" property to false
- // prevents this.
- // All other known open-source stax parsers (and the bea ref impl)
- // have thread-safe factories.
- inputFactory.setProperty("reuse-instance", Boolean.FALSE);
- }
- catch (IllegalArgumentException ex) {
- // Other implementations will likely throw this exception since "reuse-instance"
- // isimplementation specific.
- log.debug("Unable to set the 'reuse-instance' property for the input chain: " + inputFactory);
- }
- inputFactory.setXMLReporter(xmllog);
-
- final SolrParams p = SolrParams.toSolrParams(args);
- this.xsltCacheLifetimeSeconds = p.getInt(XSLT_CACHE_PARAM,XSLT_CACHE_DEFAULT);
- log.info("xsltCacheLifetimeSeconds=" + xsltCacheLifetimeSeconds);
- }
-
- @Override
- protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
- return new XsltXMLLoader(processor, inputFactory, xsltCacheLifetimeSeconds);
+ setAssumeContentType("application/xml");
+ log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
}
//////////////////////// SolrInfoMBeans methods //////////////////////
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java Tue May 8 21:06:10 2012
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.loader;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import java.io.*;
+
+public class CSVLoader extends ContentStreamLoader {
+ @Override
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp,
+ ContentStream stream, UpdateRequestProcessor processor) throws Exception {
+ new SingleThreadedCSVLoader(req,processor).load(req, rsp, stream, processor);
+ }
+}
+
+class SingleThreadedCSVLoader extends CSVLoaderBase {
+ SingleThreadedCSVLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
+ super(req, processor);
+ }
+
+ @Override
+ void addDoc(int line, String[] vals) throws IOException {
+ templateAdd.clear();
+ SolrInputDocument doc = new SolrInputDocument();
+ doAdd(line, vals, doc, templateAdd);
+ }
+}
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java Tue May 8 21:06:10 2012
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.loader;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.update.*;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.internal.csv.CSVStrategy;
+import org.apache.solr.internal.csv.CSVParser;
+import org.apache.commons.io.IOUtils;
+
+import java.util.regex.Pattern;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.io.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class CSVLoaderBase extends ContentStreamLoader {
+ public static final String SEPARATOR="separator";
+ public static final String FIELDNAMES="fieldnames";
+ public static final String HEADER="header";
+ public static final String SKIP="skip";
+ public static final String SKIPLINES="skipLines";
+ public static final String MAP="map";
+ public static final String TRIM="trim";
+ public static final String EMPTY="keepEmpty";
+ public static final String SPLIT="split";
+ public static final String ENCAPSULATOR="encapsulator";
+ public static final String ESCAPE="escape";
+ public static final String OVERWRITE="overwrite";
+ public static final String LITERALS_PREFIX = "literal.";
+
+ private static Pattern colonSplit = Pattern.compile(":");
+ private static Pattern commaSplit = Pattern.compile(",");
+
+ public static Logger log = LoggerFactory.getLogger(CSVLoaderBase.class);
+
+ final IndexSchema schema;
+ final SolrParams params;
+ final CSVStrategy strategy;
+ final UpdateRequestProcessor processor;
+
+ // hashmap to save any literal fields and their values
+ HashMap <SchemaField, String> literals;
+ String[] fieldnames;
+ SchemaField[] fields;
+ CSVLoaderBase.FieldAdder[] adders;
+
+ int skipLines; // number of lines to skip at start of file
+
+ final AddUpdateCommand templateAdd;
+
+
+
+ /** Add a field to a document unless it's zero length.
+ * The FieldAdder hierarchy handles all the complexity of
+ * further transforming or splitting field values to keep the
+ * main logic loop clean. All implementations of add() must be
+ * MT-safe!
+ */
+ private class FieldAdder {
+ void add(SolrInputDocument doc, int line, int column, String val) {
+ if (val.length() > 0) {
+ doc.addField(fields[column].getName(),val,1.0f);
+ }
+ }
+ }
+
+ /** add zero length fields */
+ private class FieldAdderEmpty extends CSVLoaderBase.FieldAdder {
+ @Override
+ void add(SolrInputDocument doc, int line, int column, String val) {
+ doc.addField(fields[column].getName(),val,1.0f);
+ }
+ }
+
+ /** trim fields */
+ private class FieldTrimmer extends CSVLoaderBase.FieldAdder {
+ private final CSVLoaderBase.FieldAdder base;
+ FieldTrimmer(CSVLoaderBase.FieldAdder base) { this.base=base; }
+ @Override
+ void add(SolrInputDocument doc, int line, int column, String val) {
+ base.add(doc, line, column, val.trim());
+ }
+ }
+
+ /** map a single value.
+ * for just a couple of mappings, this is probably faster than
+ * using a HashMap.
+ */
+ private class FieldMapperSingle extends CSVLoaderBase.FieldAdder {
+ private final String from;
+ private final String to;
+ private final CSVLoaderBase.FieldAdder base;
+ FieldMapperSingle(String from, String to, CSVLoaderBase.FieldAdder base) {
+ this.from=from;
+ this.to=to;
+ this.base=base;
+ }
+ @Override
+ void add(SolrInputDocument doc, int line, int column, String val) {
+ if (from.equals(val)) val=to;
+ base.add(doc,line,column,val);
+ }
+ }
+
+ /** Split a single value into multiple values based on
+ * a CSVStrategy.
+ */
+ private class FieldSplitter extends CSVLoaderBase.FieldAdder {
+ private final CSVStrategy strategy;
+ private final CSVLoaderBase.FieldAdder base;
+ FieldSplitter(CSVStrategy strategy, CSVLoaderBase.FieldAdder base) {
+ this.strategy = strategy;
+ this.base = base;
+ }
+
+ @Override
+ void add(SolrInputDocument doc, int line, int column, String val) {
+ CSVParser parser = new CSVParser(new StringReader(val), strategy);
+ try {
+ String[] vals = parser.getLine();
+ if (vals!=null) {
+ for (String v: vals) base.add(doc,line,column,v);
+ } else {
+ base.add(doc,line,column,val);
+ }
+ } catch (IOException e) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,e);
+ }
+ }
+ }
+
+
+ String errHeader="CSVLoader:";
+
+ CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
+ this.processor = processor;
+ this.params = req.getParams();
+ schema = req.getSchema();
+ this.literals = new HashMap<SchemaField, String>();
+
+ templateAdd = new AddUpdateCommand(req);
+ templateAdd.overwrite=params.getBool(OVERWRITE,true);
+ templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
+
+ strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
+ String sep = params.get(SEPARATOR);
+ if (sep!=null) {
+ if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
+ strategy.setDelimiter(sep.charAt(0));
+ }
+
+ String encapsulator = params.get(ENCAPSULATOR);
+ if (encapsulator!=null) {
+ if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
+ }
+
+ String escape = params.get(ESCAPE);
+ if (escape!=null) {
+ if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
+ }
+
+ // if only encapsulator or escape is set, disable the other escaping mechanism
+ if (encapsulator == null && escape != null) {
+ strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
+ strategy.setEscape(escape.charAt(0));
+ } else {
+ if (encapsulator != null) {
+ strategy.setEncapsulator(encapsulator.charAt(0));
+ }
+ if (escape != null) {
+ char ch = escape.charAt(0);
+ strategy.setEscape(ch);
+ if (ch == '\\') {
+ // If the escape is the standard backslash, then also enable
+ // unicode escapes (it's harmless since 'u' would not otherwise
+ // be escaped.
+ strategy.setUnicodeEscapeInterpretation(true);
+ }
+ }
+ }
+
+ String fn = params.get(FIELDNAMES);
+ fieldnames = fn != null ? commaSplit.split(fn,-1) : null;
+
+ Boolean hasHeader = params.getBool(HEADER);
+
+ skipLines = params.getInt(SKIPLINES,0);
+
+ if (fieldnames==null) {
+ if (null == hasHeader) {
+ // assume the file has the headers if they aren't supplied in the args
+ hasHeader=true;
+ } else if (!hasHeader) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"CSVLoader: must specify fieldnames=<fields>* or header=true");
+ }
+ } else {
+ // if the fieldnames were supplied and the file has a header, we need to
+ // skip over that header.
+ if (hasHeader!=null && hasHeader) skipLines++;
+
+ prepareFields();
+ }
+ }
+
+ /** create the FieldAdders that control how each field is indexed */
+ void prepareFields() {
+ // Possible future optimization: for really rapid incremental indexing
+ // from a POST, one could cache all of this setup info based on the params.
+ // The link from FieldAdder to this would need to be severed for that to happen.
+
+ fields = new SchemaField[fieldnames.length];
+ adders = new CSVLoaderBase.FieldAdder[fieldnames.length];
+ String skipStr = params.get(SKIP);
+ List<String> skipFields = skipStr==null ? null : StrUtils.splitSmart(skipStr,',');
+
+ CSVLoaderBase.FieldAdder adder = new CSVLoaderBase.FieldAdder();
+ CSVLoaderBase.FieldAdder adderKeepEmpty = new CSVLoaderBase.FieldAdderEmpty();
+
+ for (int i=0; i<fields.length; i++) {
+ String fname = fieldnames[i];
+ // to skip a field, leave the entries in fields and addrs null
+ if (fname.length()==0 || (skipFields!=null && skipFields.contains(fname))) continue;
+
+ fields[i] = schema.getField(fname);
+ boolean keepEmpty = params.getFieldBool(fname,EMPTY,false);
+ adders[i] = keepEmpty ? adderKeepEmpty : adder;
+
+ // Order that operations are applied: split -> trim -> map -> add
+ // so create in reverse order.
+ // Creation of FieldAdders could be optimized and shared among fields
+
+ String[] fmap = params.getFieldParams(fname,MAP);
+ if (fmap!=null) {
+ for (String mapRule : fmap) {
+ String[] mapArgs = colonSplit.split(mapRule,-1);
+ if (mapArgs.length!=2)
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Map rules must be of the form 'from:to' ,got '"+mapRule+"'");
+ adders[i] = new CSVLoaderBase.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
+ }
+ }
+
+ if (params.getFieldBool(fname,TRIM,false)) {
+ adders[i] = new CSVLoaderBase.FieldTrimmer(adders[i]);
+ }
+
+ if (params.getFieldBool(fname,SPLIT,false)) {
+ String sepStr = params.getFieldParam(fname,SEPARATOR);
+ char fsep = sepStr==null || sepStr.length()==0 ? ',' : sepStr.charAt(0);
+ String encStr = params.getFieldParam(fname,ENCAPSULATOR);
+ char fenc = encStr==null || encStr.length()==0 ? (char)-2 : encStr.charAt(0);
+ String escStr = params.getFieldParam(fname,ESCAPE);
+ char fesc = escStr==null || escStr.length()==0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0);
+
+ CSVStrategy fstrat = new CSVStrategy(fsep,fenc,CSVStrategy.COMMENTS_DISABLED,fesc, false, false, false, false);
+ adders[i] = new CSVLoaderBase.FieldSplitter(fstrat, adders[i]);
+ }
+ }
+ // look for any literal fields - literal.foo=xyzzy
+ Iterator<String> paramNames = params.getParameterNamesIterator();
+ while (paramNames.hasNext()) {
+ String pname = paramNames.next();
+ if (!pname.startsWith(LITERALS_PREFIX)) continue;
+
+ String name = pname.substring(LITERALS_PREFIX.length());
+ SchemaField sf = schema.getFieldOrNull(name);
+ if(sf == null)
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
+ literals.put(sf, params.get(pname));
+ }
+ }
+
+ private void input_err(String msg, String[] line, int lineno) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(errHeader).append(", line=").append(lineno).append(",").append(msg).append("\n\tvalues={");
+ for (String val: line) {
+ sb.append("'").append(val).append("',"); }
+ sb.append('}');
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString());
+ }
+
+ private void input_err(String msg, String[] lines, int lineNo, Throwable e) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(errHeader).append(", line=").append(lineNo).append(",").append(msg).append("\n\tvalues={");
+ if (lines != null) {
+ for (String val : lines) {
+ sb.append("'").append(val).append("',");
+ }
+ } else {
+ sb.append("NO LINES AVAILABLE");
+ }
+ sb.append('}');
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString(), e);
+ }
+
+ /** load the CSV input */
+ @Override
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws IOException {
+ errHeader = "CSVLoader: input=" + stream.getSourceInfo();
+ Reader reader = null;
+ try {
+ reader = stream.getReader();
+ if (skipLines>0) {
+ if (!(reader instanceof BufferedReader)) {
+ reader = new BufferedReader(reader);
+ }
+ BufferedReader r = (BufferedReader)reader;
+ for (int i=0; i<skipLines; i++) {
+ r.readLine();
+ }
+ }
+
+ CSVParser parser = new CSVParser(reader, strategy);
+
+ // parse the fieldnames from the header of the file
+ if (fieldnames==null) {
+ fieldnames = parser.getLine();
+ if (fieldnames==null) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
+ }
+ prepareFields();
+ }
+
+ // read the rest of the CSV file
+ for(;;) {
+ int line = parser.getLineNumber(); // for error reporting in MT mode
+ String[] vals = null;
+ try {
+ vals = parser.getLine();
+ } catch (IOException e) {
+ //Catch the exception and rethrow it with more line information
+ input_err("can't read line: " + line, null, line, e);
+ }
+ if (vals==null) break;
+
+ if (vals.length != fields.length) {
+ input_err("expected "+fields.length+" values but got "+vals.length, vals, line);
+ }
+
+ addDoc(line,vals);
+ }
+ } finally{
+ if (reader != null) {
+ IOUtils.closeQuietly(reader);
+ }
+ }
+ }
+
+ /** called for each line of values (document) */
+ abstract void addDoc(int line, String[] vals) throws IOException;
+
+ /** this must be MT safe... may be called concurrently from multiple threads. */
+ void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template) throws IOException {
+ // the line number is passed simply for error reporting in MT mode.
+ // first, create the lucene document
+ for (int i=0; i<vals.length; i++) {
+ if (fields[i]==null) continue; // ignore this field
+ String val = vals[i];
+ adders[i].add(doc, line, i, val);
+ }
+
+ // add any literals
+ for (SchemaField sf : literals.keySet()) {
+ String fn = sf.getName();
+ String val = literals.get(sf);
+ doc.addField(fn, val);
+ }
+
+ template.solrDoc = doc;
+ processor.processAdd(template);
+ }
+}
\ No newline at end of file
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java Tue May 8 21:06:10 2012
@@ -0,0 +1,57 @@
+package org.apache.solr.handler.loader;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+
+/**
+ * Load a {@link org.apache.solr.common.util.ContentStream} into Solr
+ *
+ * This should be thread safe and can be called from multiple threads
+ */
+public abstract class ContentStreamLoader {
+
+ /**
+ * This should be called once for each RequestHandler
+ */
+ public ContentStreamLoader init(SolrParams args) {
+ return this;
+ }
+
+ public String getDefaultWT() {
+ return null;
+ }
+
+ /**
+ * Loaders are responsible for closing the stream
+ *
+ * @param req The input {@link org.apache.solr.request.SolrQueryRequest}
+ * @param rsp The response, in case the Loader wishes to add anything
+ * @param stream The {@link org.apache.solr.common.util.ContentStream} to add
+ * @param processor The {@link UpdateRequestProcessor} to use
+ */
+ public abstract void load(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ ContentStream stream,
+ UpdateRequestProcessor processor) throws Exception;
+}