You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2012/05/08 23:06:13 UTC

svn commit: r1335768 [1/3] - in /lucene/dev/trunk/solr: contrib/clustering/src/test-files/clustering/solr/conf/ contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/ contrib/dataimporthandler/src/test-files/dih/solr/conf/ contrib/extract...

Author: ryan
Date: Tue May  8 21:06:10 2012
New Revision: 1335768

URL: http://svn.apache.org/viewvc?rev=1335768&view=rev
Log:
SOLR-2857: add UpdateRequestHandler that handles xml,csv,json, and javabin

Added:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java   (with props)
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java   (with props)
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java   (with props)
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java   (with props)
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java   (with props)
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
      - copied, changed from r1335654, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java   (with props)
Removed:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonLoader.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XMLLoader.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltXMLLoader.java
Modified:
    lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml
    lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml
    lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
    lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
    lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
    lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml
    lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
    lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
    lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-enableplugin.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-functionquery.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master1-keepOneBackup.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master1.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master2.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-master3.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-slave.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-solcoreproperties.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig-tlog.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/conf/solrconfig_perf.xml
    lucene/dev/trunk/solr/core/src/test-files/solr/crazy-path-to-config.xml
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/ConvertedLegacyTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/RequestHandlersTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/BinaryUpdateRequestHandlerTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/CSVRequestHandlerTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/XsltUpdateRequestHandlerTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/CacheHeaderTestBase.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/UpdateParamsTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/SignatureUpdateProcessorFactoryTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactoryTest.java
    lucene/dev/trunk/solr/example/example-DIH/solr/db/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/example-DIH/solr/mail/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/example-DIH/solr/rss/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/example-DIH/solr/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/example-DIH/solr/tika/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/request/ContentStreamUpdateRequest.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
    lucene/dev/trunk/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml
    lucene/dev/trunk/solr/solrj/src/test-files/solrj/solr/shared/conf/solrconfig.xml
    lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
    lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest.java
    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java

Modified: lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/clustering/src/test-files/clustering/solr/conf/solrconfig.xml Tue May  8 21:06:10 2012
@@ -361,10 +361,7 @@
        
        To enable solr1.1 behavior, remove the /update handler or change its path
     -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
-
-  <!-- CSV update handler, loaded on demand -->
-  <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" />
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
 
 
   <!-- 

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/solr/conf/dataimport-solrconfig.xml Tue May  8 21:06:10 2012
@@ -296,7 +296,7 @@
        "update.processor.class" is the class name for the UpdateRequestProcessor.  It is initalized
        only once.  This can not be changed for each request.
     -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+  <requestHandler name="/update" class="solr.UpdateRequestHandler">
     <!--
     <str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
     -->

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/contentstream-solrconfig.xml Tue May  8 21:06:10 2012
@@ -300,7 +300,7 @@
        "update.processor.class" is the class name for the UpdateRequestProcessor.  It is initalized
        only once.  This can not be changed for each request.
     -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+  <requestHandler name="/update" class="solr.UpdateRequestHandler">
     <!--
     <str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
     -->

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-nodatasource-solrconfig.xml Tue May  8 21:06:10 2012
@@ -298,7 +298,7 @@
        "update.processor.class" is the class name for the UpdateRequestProcessor.  It is initalized
        only once.  This can not be changed for each request.
     -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+  <requestHandler name="/update" class="solr.UpdateRequestHandler">
     <!--
     <str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
     -->

Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/dih/solr/conf/dataimport-solrconfig.xml Tue May  8 21:06:10 2012
@@ -296,7 +296,7 @@
        "update.processor.class" is the class name for the UpdateRequestProcessor.  It is initalized
        only once.  This can not be changed for each request.
     -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" >
+  <requestHandler name="/update" class="solr.UpdateRequestHandler" >
     <!--
     <str name="update.processor.class">org.apache.solr.handler.UpdateRequestProcessor</str>
     -->

Modified: lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue May  8 21:06:10 2012
@@ -28,7 +28,7 @@ import org.apache.solr.common.params.Upd
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.handler.ContentStreamLoader;
+import org.apache.solr.handler.loader.ContentStreamLoader;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.schema.IndexSchema;
@@ -134,8 +134,8 @@ public class ExtractingDocumentLoader ex
    * @throws java.io.IOException
    */
   @Override
-  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws IOException {
-    errHeader = "ExtractingDocumentLoader: " + stream.getSourceInfo();
+  public void load(SolrQueryRequest req, SolrQueryResponse rsp,
+      ContentStream stream, UpdateRequestProcessor processor) throws Exception {
     Parser parser = null;
     String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);
     if (streamType != null) {
@@ -240,6 +240,4 @@ public class ExtractingDocumentLoader ex
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stream type of " + streamType + " didn't match any known parsers.  Please supply the " + ExtractingParams.STREAM_TYPE + " parameter.");
     }
   }
-
-
 }

Modified: lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java Tue May  8 21:06:10 2012
@@ -26,7 +26,7 @@ import org.apache.solr.request.SolrQuery
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.solr.util.plugin.SolrCoreAware;
 import org.apache.solr.handler.ContentStreamHandlerBase;
-import org.apache.solr.handler.ContentStreamLoader;
+import org.apache.solr.handler.loader.ContentStreamLoader;
 import org.apache.tika.config.TikaConfig;
 import org.apache.tika.mime.MimeTypeException;
 import org.slf4j.Logger;

Modified: lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/test-files/extraction/solr/conf/solrconfig.xml Tue May  8 21:06:10 2012
@@ -194,10 +194,7 @@
     </lst>
   </requestHandler>
 
-  <requestHandler name="/update"     class="solr.XmlUpdateRequestHandler"          />
-  <requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy">
-  	<bool name="httpCaching">false</bool>
-  </requestHandler>
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
   
   <requestHandler name="/update/extract" class="org.apache.solr.handler.extraction.ExtractingRequestHandler"/>
 

Modified: lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Tue May  8 21:06:10 2012
@@ -301,7 +301,7 @@ public class ExtractingRequestHandlerTes
     BufferingRequestProcessor p = new BufferingRequestProcessor(null);
 
     ExtractingDocumentLoader loader = (ExtractingDocumentLoader) handler.newLoader(req, p);
-    loader.load(req, rsp, new ContentStreamBase.FileStream(getFile("extraction/version_control.txt")));
+    loader.load(req, rsp, new ContentStreamBase.FileStream(getFile("extraction/version_control.txt")),p);
 
     AddUpdateCommand add = p.addCommands.get(0);
     assertEquals(200, add.commitWithin);

Modified: lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml (original)
+++ lucene/dev/trunk/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml Tue May  8 21:06:10 2012
@@ -55,7 +55,7 @@
     </httpCaching>
   </requestDispatcher>
 
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler">
+  <requestHandler name="/update" class="solr.UpdateRequestHandler" >
     <lst name="defaults">
       <str name="update.chain">lang_id</str>
     </lst>

Modified: lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/solr/conf/solrconfig.xml Tue May  8 21:06:10 2012
@@ -705,15 +705,13 @@
     code. To enable solr1.1 behavior, remove the /update handler or
     change its path
   -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler">
+  <requestHandler name="/update" class="solr.UpdateRequestHandler" >
     <lst name="defaults">
       <str name="update.chain">uima</str>
     </lst>
   </requestHandler>
 
 
-  <requestHandler name="/update/javabin" class="solr.BinaryUpdateRequestHandler" />
-
   <!--
     Analysis request handler. Since Solr 1.3. Use to return how a
     document is analyzed. Useful for debugging and as a token server for
@@ -803,7 +801,7 @@
     signatureField and Solr will maintain uniqueness based on that
     anyway. You have to link the chain to an update handler above to use
     it ie: <requestHandler name="/update
-    "class="solr.XmlUpdateRequestHandler"> <lst name="defaults"> <str
+    "class="solr.UpdateRequestHandler"> <lst name="defaults"> <str
     name="update.chain">dedupe</str> </lst> </requestHandler>
   -->
 

Modified: lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml Tue May  8 21:06:10 2012
@@ -705,9 +705,7 @@
     code. To enable solr1.1 behavior, remove the /update handler or
     change its path
   -->
-  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler"/>
-
-  <requestHandler name="/update/javabin" class="solr.BinaryUpdateRequestHandler" />
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
 
   <!--
     Analysis request handler. Since Solr 1.3. Use to return how a
@@ -798,7 +796,7 @@
     signatureField and Solr will maintain uniqueness based on that
     anyway. You have to link the chain to an update handler above to use
     it ie: <requestHandler name="/update
-    "class="solr.XmlUpdateRequestHandler"> <lst name="defaults"> <str
+    "class="solr.UpdateRequestHandler"> <lst name="defaults"> <str
     name="update.chain">dedupe</str> </lst> </requestHandler>
   -->
 

Modified: lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/trunk/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Tue May  8 21:06:10 2012
@@ -31,7 +31,7 @@ import org.apache.solr.common.params.Upd
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.XmlUpdateRequestHandler;
+import org.apache.solr.handler.UpdateRequestHandler;
 import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
@@ -192,7 +192,7 @@ public class UIMAUpdateRequestProcessorT
     SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), (SolrParams) mmparams) {
     };
 
-    XmlUpdateRequestHandler handler = new XmlUpdateRequestHandler();
+    UpdateRequestHandler handler = new UpdateRequestHandler();
     handler.init(null);
     ArrayList<ContentStream> streams = new ArrayList<ContentStream>(2);
     streams.add(new ContentStreamBase.StringStream(doc));

Modified: lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/contrib/velocity/src/test-files/velocity/solr/conf/solrconfig.xml Tue May  8 21:06:10 2012
@@ -83,18 +83,8 @@
      </lst>
     </requestHandler>
 
-
-  <requestHandler name="/update"
-                  class="solr.XmlUpdateRequestHandler">
-    </requestHandler>
-  <!-- Binary Update Request Handler
-       http://wiki.apache.org/solr/javabin
-    -->
-  <requestHandler name="/update/javabin" 
-                  class="solr.BinaryUpdateRequestHandler" />
-  <requestHandler name="/update/json"
-                  class="solr.JsonUpdateRequestHandler" 
-                  startup="lazy" />
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
+  
   <!--
      <queryResponseWriter name="xml" 
                           default="true"

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/BinaryUpdateRequestHandler.java Tue May  8 21:06:10 2012
@@ -17,128 +17,24 @@
 
 package org.apache.solr.handler;
 
-import org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.common.util.FastInputStream;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.AddUpdateCommand;
-import org.apache.solr.update.DeleteUpdateCommand;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
+import org.apache.solr.common.util.NamedList;
 
 /**
  * Update handler which uses the JavaBin format
  *
- *
  * @see org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec
  * @see org.apache.solr.common.util.JavaBinCodec
- * @since solr 1.4
+ * 
+ * use {@link UpdateRequestHandler}
  */
-public class BinaryUpdateRequestHandler extends ContentStreamHandlerBase {
-
+@Deprecated
+public class BinaryUpdateRequestHandler extends UpdateRequestHandler {
 
   @Override
-  protected ContentStreamLoader newLoader(SolrQueryRequest req, final UpdateRequestProcessor processor) {
-    return new ContentStreamLoader() {
-      @Override
-      public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws Exception {
-        InputStream is = null;
-        try {
-          is = stream.getStream();
-          parseAndLoadDocs(req, rsp, is, processor);
-        } finally {
-          if(is != null) {
-            is.close();
-          }
-        }
-      }
-    };
-  }
-
-  private void parseAndLoadDocs(final SolrQueryRequest req, SolrQueryResponse rsp, InputStream stream,
-                                final UpdateRequestProcessor processor) throws IOException {
-    UpdateRequest update = null;
-    JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler() {
-      private AddUpdateCommand addCmd = null;
-
-      public void update(SolrInputDocument document, UpdateRequest updateRequest) {
-        if (document == null) {
-          // Perhaps commit from the parameters
-          try {
-            RequestHandlerUtils.handleCommit(req, processor, updateRequest.getParams(), false);
-            RequestHandlerUtils.handleRollback(req, processor, updateRequest.getParams(), false);
-          } catch (IOException e) {
-            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ERROR handling commit/rollback");
-          }
-          return;
-        }
-        if (addCmd == null) {
-          addCmd = getAddCommand(req, updateRequest.getParams());
-        }
-        addCmd.solrDoc = document;
-        try {
-          processor.processAdd(addCmd);
-          addCmd.clear();
-        } catch (IOException e) {
-          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ERROR adding document " + document);
-        }
-      }
-    };
-    FastInputStream in = FastInputStream.wrap(stream);
-    for (; ; ) {
-      try {
-        update = new JavaBinUpdateRequestCodec().unmarshal(in, handler);
-      } catch (EOFException e) {
-        break; // this is expected
-      } catch (Exception e) {
-        log.error("Exception while processing update request", e);
-        break;
-      }
-      if (update.getDeleteById() != null || update.getDeleteQuery() != null) {
-        delete(req, update, processor);
-      }
-    }
-  }
-
-  private AddUpdateCommand getAddCommand(SolrQueryRequest req, SolrParams params) {
-    AddUpdateCommand addCmd = new AddUpdateCommand(req);
-
-    addCmd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
-    addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
-    return addCmd;
-  }
-
-  private void delete(SolrQueryRequest req, UpdateRequest update, UpdateRequestProcessor processor) throws IOException {
-    SolrParams params = update.getParams();
-    DeleteUpdateCommand delcmd = new DeleteUpdateCommand(req);
-    if(params != null) {
-      delcmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
-    }
-    
-    if(update.getDeleteById() != null) {
-      for (String s : update.getDeleteById()) {
-        delcmd.id = s;
-        processor.processDelete(delcmd);
-      }
-      delcmd.id = null;
-    }
-    
-    if(update.getDeleteQuery() != null) {
-      for (String s : update.getDeleteQuery()) {
-        delcmd.query = s;
-        processor.processDelete(delcmd);
-      }
-    }
+  public void init(NamedList args) {
+    super.init(args);
+    setAssumeContentType("application/javabin");
+    log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
   }
 
   @Override

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue May  8 21:06:10 2012
@@ -17,40 +17,19 @@
 
 package org.apache.solr.handler;
 
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.update.*;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.internal.csv.CSVStrategy;
-import org.apache.solr.internal.csv.CSVParser;
-import org.apache.commons.io.IOUtils;
-
-import java.util.regex.Pattern;
-import java.util.List;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.io.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.solr.common.util.NamedList;
 
 /**
- *
+ * use {@link UpdateRequestHandler}
  */
-
-public class CSVRequestHandler extends ContentStreamHandlerBase {
-	public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
+@Deprecated
+public class CSVRequestHandler extends UpdateRequestHandler {
 
   @Override
-  protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
-    return new SingleThreadedCSVLoader(req, processor);
+  public void init(NamedList args) {
+    super.init(args);
+    setAssumeContentType("application/csv");
+    log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
   }
 
   //////////////////////// SolrInfoMBeans methods //////////////////////
@@ -66,376 +45,6 @@ public class CSVRequestHandler extends C
 }
 
 
-abstract class CSVLoader extends ContentStreamLoader {
-  public static final String SEPARATOR="separator";
-  public static final String FIELDNAMES="fieldnames";
-  public static final String HEADER="header";
-  public static final String SKIP="skip";
-  public static final String SKIPLINES="skipLines";
-  public static final String MAP="map";
-  public static final String TRIM="trim";
-  public static final String EMPTY="keepEmpty";
-  public static final String SPLIT="split";
-  public static final String ENCAPSULATOR="encapsulator";
-  public static final String ESCAPE="escape";
-  public static final String OVERWRITE="overwrite";
-  public static final String LITERALS_PREFIX = "literal.";
-
-  private static Pattern colonSplit = Pattern.compile(":");
-  private static Pattern commaSplit = Pattern.compile(",");
-  
-  public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
-
-  final IndexSchema schema;
-  final SolrParams params;
-  final CSVStrategy strategy;
-  final UpdateRequestProcessor processor;
-
-  // hashmap to save any literal fields and their values
-  HashMap <SchemaField, String> literals;
-  String[] fieldnames;
-  SchemaField[] fields;
-  CSVLoader.FieldAdder[] adders;
-
-  int skipLines;    // number of lines to skip at start of file
-
-  final AddUpdateCommand templateAdd;
-
-
-
-  /** Add a field to a document unless it's zero length.
-   * The FieldAdder hierarchy handles all the complexity of
-   * further transforming or splitting field values to keep the
-   * main logic loop clean.  All implementations of add() must be
-   * MT-safe!
-   */
-  private class FieldAdder {
-    void add(SolrInputDocument doc, int line, int column, String val) {
-      if (val.length() > 0) {
-        doc.addField(fields[column].getName(),val,1.0f);
-      }
-    }
-  }
-
-  /** add zero length fields */
-  private class FieldAdderEmpty extends CSVLoader.FieldAdder {
-    @Override
-    void add(SolrInputDocument doc, int line, int column, String val) {
-      doc.addField(fields[column].getName(),val,1.0f);
-    }
-  }
-
-  /** trim fields */
-  private class FieldTrimmer extends CSVLoader.FieldAdder {
-    private final CSVLoader.FieldAdder base;
-    FieldTrimmer(CSVLoader.FieldAdder base) { this.base=base; }
-    @Override
-    void add(SolrInputDocument doc, int line, int column, String val) {
-      base.add(doc, line, column, val.trim());
-    }
-  }
-
-  /** map a single value.
-   * for just a couple of mappings, this is probably faster than
-   * using a HashMap.
-   */
- private class FieldMapperSingle extends CSVLoader.FieldAdder {
-   private final String from;
-   private final String to;
-   private final CSVLoader.FieldAdder base;
-   FieldMapperSingle(String from, String to, CSVLoader.FieldAdder base) {
-     this.from=from;
-     this.to=to;
-     this.base=base;
-   }
-    @Override
-    void add(SolrInputDocument doc, int line, int column, String val) {
-      if (from.equals(val)) val=to;
-      base.add(doc,line,column,val);
-    }
- }
-
-  /** Split a single value into multiple values based on
-   * a CSVStrategy.
-   */
-  private class FieldSplitter extends CSVLoader.FieldAdder {
-    private final CSVStrategy strategy;
-    private final CSVLoader.FieldAdder base;
-    FieldSplitter(CSVStrategy strategy, CSVLoader.FieldAdder base) {
-      this.strategy = strategy;
-      this.base = base;
-    }
-
-    @Override
-    void add(SolrInputDocument doc, int line, int column, String val) {
-      CSVParser parser = new CSVParser(new StringReader(val), strategy);
-      try {
-        String[] vals = parser.getLine();
-        if (vals!=null) {
-          for (String v: vals) base.add(doc,line,column,v);
-        } else {
-          base.add(doc,line,column,val);
-        }
-      } catch (IOException e) {
-        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,e);
-      }
-    }
-  }
-
-
-  String errHeader="CSVLoader:";
-
-  CSVLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
-    this.processor = processor;
-    this.params = req.getParams();
-    schema = req.getSchema();
-    this.literals = new HashMap<SchemaField, String>();
-
-    templateAdd = new AddUpdateCommand(req);
-    templateAdd.overwrite=params.getBool(OVERWRITE,true);
-    templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
-    
-    strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
-    String sep = params.get(SEPARATOR);
-    if (sep!=null) {
-      if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
-      strategy.setDelimiter(sep.charAt(0));
-    }
-
-    String encapsulator = params.get(ENCAPSULATOR);
-    if (encapsulator!=null) {
-      if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
-    }
-
-    String escape = params.get(ESCAPE);
-    if (escape!=null) {
-      if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
-    }
-
-    // if only encapsulator or escape is set, disable the other escaping mechanism
-    if (encapsulator == null && escape != null) {
-      strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);     
-      strategy.setEscape(escape.charAt(0));
-    } else {
-      if (encapsulator != null) {
-        strategy.setEncapsulator(encapsulator.charAt(0));
-      }
-      if (escape != null) {
-        char ch = escape.charAt(0);
-        strategy.setEscape(ch);
-        if (ch == '\\') {
-          // If the escape is the standard backslash, then also enable
-          // unicode escapes (it's harmless since 'u' would not otherwise
-          // be escaped.                    
-          strategy.setUnicodeEscapeInterpretation(true);
-        }
-      }
-    }
-
-    String fn = params.get(FIELDNAMES);
-    fieldnames = fn != null ? commaSplit.split(fn,-1) : null;
-
-    Boolean hasHeader = params.getBool(HEADER);
-
-    skipLines = params.getInt(SKIPLINES,0);
-
-    if (fieldnames==null) {
-      if (null == hasHeader) {
-        // assume the file has the headers if they aren't supplied in the args
-        hasHeader=true;
-      } else if (!hasHeader) {
-        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"CSVLoader: must specify fieldnames=<fields>* or header=true");
-      }
-    } else {
-      // if the fieldnames were supplied and the file has a header, we need to
-      // skip over that header.
-      if (hasHeader!=null && hasHeader) skipLines++;
-
-      prepareFields();
-    }
-  }
-
-  /** create the FieldAdders that control how each field  is indexed */
-  void prepareFields() {
-    // Possible future optimization: for really rapid incremental indexing
-    // from a POST, one could cache all of this setup info based on the params.
-    // The link from FieldAdder to this would need to be severed for that to happen.
-
-    fields = new SchemaField[fieldnames.length];
-    adders = new CSVLoader.FieldAdder[fieldnames.length];
-    String skipStr = params.get(SKIP);
-    List<String> skipFields = skipStr==null ? null : StrUtils.splitSmart(skipStr,',');
-
-    CSVLoader.FieldAdder adder = new CSVLoader.FieldAdder();
-    CSVLoader.FieldAdder adderKeepEmpty = new CSVLoader.FieldAdderEmpty();
-
-    for (int i=0; i<fields.length; i++) {
-      String fname = fieldnames[i];
-      // to skip a field, leave the entries in fields and addrs null
-      if (fname.length()==0 || (skipFields!=null && skipFields.contains(fname))) continue;
-
-      fields[i] = schema.getField(fname);
-      boolean keepEmpty = params.getFieldBool(fname,EMPTY,false);
-      adders[i] = keepEmpty ? adderKeepEmpty : adder;
-
-      // Order that operations are applied: split -> trim -> map -> add
-      // so create in reverse order.
-      // Creation of FieldAdders could be optimized and shared among fields
-
-      String[] fmap = params.getFieldParams(fname,MAP);
-      if (fmap!=null) {
-        for (String mapRule : fmap) {
-          String[] mapArgs = colonSplit.split(mapRule,-1);
-          if (mapArgs.length!=2)
-            throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Map rules must be of the form 'from:to' ,got '"+mapRule+"'");
-          adders[i] = new CSVLoader.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
-        }
-      }
- 
-      if (params.getFieldBool(fname,TRIM,false)) {
-        adders[i] = new CSVLoader.FieldTrimmer(adders[i]);
-      }
-
-      if (params.getFieldBool(fname,SPLIT,false)) {
-        String sepStr = params.getFieldParam(fname,SEPARATOR);
-        char fsep = sepStr==null || sepStr.length()==0 ? ',' : sepStr.charAt(0);
-        String encStr = params.getFieldParam(fname,ENCAPSULATOR);
-        char fenc = encStr==null || encStr.length()==0 ? (char)-2 : encStr.charAt(0);
-        String escStr = params.getFieldParam(fname,ESCAPE);
-        char fesc = escStr==null || escStr.length()==0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0);
-
-        CSVStrategy fstrat = new CSVStrategy(fsep,fenc,CSVStrategy.COMMENTS_DISABLED,fesc, false, false, false, false);
-        adders[i] = new CSVLoader.FieldSplitter(fstrat, adders[i]);
-      }
-    }
-    // look for any literal fields - literal.foo=xyzzy
-    Iterator<String> paramNames = params.getParameterNamesIterator();
-    while (paramNames.hasNext()) {
-      String pname = paramNames.next();
-      if (!pname.startsWith(LITERALS_PREFIX)) continue;
-
-      String name = pname.substring(LITERALS_PREFIX.length());
-      SchemaField sf = schema.getFieldOrNull(name);
-      if(sf == null)
-        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
-      literals.put(sf, params.get(pname));
-    }
-  }
-
-  private void input_err(String msg, String[] line, int lineno) {
-    StringBuilder sb = new StringBuilder();
-    sb.append(errHeader).append(", line=").append(lineno).append(",").append(msg).append("\n\tvalues={");
-    for (String val: line) {
-      sb.append("'").append(val).append("',"); }
-    sb.append('}');
-    throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString());
-  }
-
-  private void input_err(String msg, String[] lines, int lineNo, Throwable e) {
-    StringBuilder sb = new StringBuilder();
-    sb.append(errHeader).append(", line=").append(lineNo).append(",").append(msg).append("\n\tvalues={");
-    if (lines != null) {
-      for (String val : lines) {
-        sb.append("'").append(val).append("',");
-      }
-    } else {
-      sb.append("NO LINES AVAILABLE");
-    }
-    sb.append('}');
-    throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString(), e);
-  }
-
-  /** load the CSV input */
-  @Override
-  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) throws IOException {
-    errHeader = "CSVLoader: input=" + stream.getSourceInfo();
-    Reader reader = null;
-    try {
-      reader = stream.getReader();
-      if (skipLines>0) {
-        if (!(reader instanceof BufferedReader)) {
-          reader = new BufferedReader(reader);
-        }
-        BufferedReader r = (BufferedReader)reader;
-        for (int i=0; i<skipLines; i++) {
-          r.readLine();
-        }
-      }
-
-      CSVParser parser = new CSVParser(reader, strategy);
-
-      // parse the fieldnames from the header of the file
-      if (fieldnames==null) {
-        fieldnames = parser.getLine();
-        if (fieldnames==null) {
-          throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
-        }
-        prepareFields();
-      }
-
-      // read the rest of the CSV file
-      for(;;) {
-        int line = parser.getLineNumber();  // for error reporting in MT mode
-        String[] vals = null;
-        try {
-          vals = parser.getLine();
-        } catch (IOException e) {
-          //Catch the exception and rethrow it with more line information
-         input_err("can't read line: " + line, null, line, e);
-        }
-        if (vals==null) break;
-
-        if (vals.length != fields.length) {
-          input_err("expected "+fields.length+" values but got "+vals.length, vals, line);
-        }
-
-        addDoc(line,vals);
-      }
-    } finally{
-      if (reader != null) {
-        IOUtils.closeQuietly(reader);
-      }
-    }
-  }
-
-  /** called for each line of values (document) */
-  abstract void addDoc(int line, String[] vals) throws IOException;
-
-  /** this must be MT safe... may be called concurrently from multiple threads. */
-  void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template) throws IOException {
-    // the line number is passed simply for error reporting in MT mode.
-    // first, create the lucene document
-    for (int i=0; i<vals.length; i++) {
-      if (fields[i]==null) continue;  // ignore this field
-      String val = vals[i];
-      adders[i].add(doc, line, i, val);
-    }
-
-    // add any literals
-    for (SchemaField sf : literals.keySet()) {
-    	String fn = sf.getName();
-    	String val = literals.get(sf);
-    	doc.addField(fn, val);
-    }
-   
-    template.solrDoc = doc;
-    processor.processAdd(template);
-  }
-
-}
-
-
-class SingleThreadedCSVLoader extends CSVLoader {
-  SingleThreadedCSVLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
-    super(req, processor);
-  }
 
-  @Override
-  void addDoc(int line, String[] vals) throws IOException {
-    templateAdd.clear();
-    SolrInputDocument doc = new SolrInputDocument();
-    doAdd(line, vals, doc, templateAdd);
-  }
-}
 
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java Tue May  8 21:06:10 2012
@@ -21,6 +21,7 @@ import org.apache.solr.common.params.Sol
 import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.loader.ContentStreamLoader;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.update.processor.UpdateRequestProcessor;
@@ -30,9 +31,9 @@ import org.slf4j.LoggerFactory;
 
 
 /**
- * Shares common code between various handlers that manipulate {@link org.apache.solr.common.util.ContentStream} objects.
- *
- **/
+ * Shares common code between various handlers that manipulate 
+ * {@link org.apache.solr.common.util.ContentStream} objects.
+ */
 public abstract class ContentStreamHandlerBase extends RequestHandlerBase {
   public static Logger log = LoggerFactory.getLogger(ContentStreamHandlerBase.class);
 
@@ -70,7 +71,7 @@ public abstract class ContentStreamHandl
       } else {
 
         for (ContentStream stream : streams) {
-          documentLoader.load(req, rsp, stream);
+          documentLoader.load(req, rsp, stream, processor);
         }
 
         // Perhaps commit from the parameters

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java Tue May  8 21:06:10 2012
@@ -18,26 +18,18 @@
 package org.apache.solr.handler;
 
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
- * Add documents to solr using the noggit JSON parser.
+ * use {@link UpdateRequestHandler}
  */
-public class JsonUpdateRequestHandler extends ContentStreamHandlerBase {
-  public static Logger log = LoggerFactory.getLogger(JsonUpdateRequestHandler.class);
-
+@Deprecated
+public class JsonUpdateRequestHandler extends UpdateRequestHandler {
 
   @Override
   public void init(NamedList args) {
     super.init(args);
-  }
-
-  @Override
-  protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
-    return new JsonLoader(req, processor);
+    setAssumeContentType("application/json");
+    log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
   }
 
   //////////////////////// SolrInfoMBeans methods //////////////////////

Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java Tue May  8 21:06:10 2012
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.loader.CSVLoader;
+import org.apache.solr.handler.loader.ContentStreamLoader;
+import org.apache.solr.handler.loader.JavabinLoader;
+import org.apache.solr.handler.loader.JsonLoader;
+import org.apache.solr.handler.loader.XMLLoader;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * UpdateHandler that uses content-type to pick the right Loader
+ */
+public class UpdateRequestHandler extends ContentStreamHandlerBase {
+  public static Logger log = LoggerFactory.getLogger(UpdateRequestHandler.class);
+
+  // XML Constants
+  public static final String ADD = "add";
+  public static final String DELETE = "delete";
+  public static final String OPTIMIZE = "optimize";
+  public static final String COMMIT = "commit";
+  public static final String ROLLBACK = "rollback";
+  public static final String WAIT_SEARCHER = "waitSearcher";
+  public static final String SOFT_COMMIT = "softCommit";
+
+  public static final String OVERWRITE = "overwrite";
+  
+  public static final String VERSION = "version";
+  
+  // NOTE: This constant is for use with the <add> XML tag, not the HTTP param with same name
+  public static final String COMMIT_WITHIN = "commitWithin";
+
+  Map<String,ContentStreamLoader> loaders = null;
+
+  ContentStreamLoader instance = new ContentStreamLoader() {
+    @Override
+    public void load(SolrQueryRequest req, SolrQueryResponse rsp,
+        ContentStream stream, UpdateRequestProcessor processor) throws Exception {
+      
+      String type = req.getParams().get(UpdateParams.ASSUME_CONTENT_TYPE);
+      if(type == null) {
+        type = stream.getContentType();
+      }
+      if( type == null ) { // Normal requests will not get here. 
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Missing ContentType");
+      }
+      int idx = type.indexOf(';');
+      if(idx>0) {
+        type = type.substring(0,idx);
+      }
+      ContentStreamLoader loader = loaders.get(type);
+      if(loader==null) {
+        throw new SolrException(ErrorCode.BAD_REQUEST, "Unsupported ContentType: "
+            +type+ "  Not in: "+loaders.keySet());
+      }
+      if(loader.getDefaultWT()!=null) {
+        setDefaultWT(req,loader);
+      }
+      loader.load(req, rsp, stream, processor);
+    }
+    
+    private void setDefaultWT(SolrQueryRequest req, ContentStreamLoader loader) {
+      SolrParams params = req.getParams();
+      if( params.get(CommonParams.WT) == null ) {
+        String wt = loader.getDefaultWT();
+        // Make sure it is a valid writer
+        if(req.getCore().getQueryResponseWriter(wt)!=null) {
+          Map<String,String> map = new HashMap<String,String>(1);
+          map.put(CommonParams.WT, wt);
+          req.setParams(SolrParams.wrapDefaults(params, 
+              new MapSolrParams(map)));
+        }
+      }
+    }
+  };
+  
+  @Override
+  public void init(NamedList args) {
+    super.init(args);
+    
+    // Since backed by a non-thread safe Map, it should not be modifiable
+    loaders = Collections.unmodifiableMap(createDefaultLoaders(args));
+  }
+  
+  protected void setAssumeContentType(String ct) {
+    if(invariants==null) {
+      Map<String,String> map = new HashMap<String,String>();
+      map.put(UpdateParams.ASSUME_CONTENT_TYPE,ct);
+      invariants = new MapSolrParams(map);
+    }
+    else {
+      ModifiableSolrParams params = new ModifiableSolrParams(invariants);
+      params.set(UpdateParams.ASSUME_CONTENT_TYPE,ct);
+      invariants = params;
+    }
+  }
+  
+  protected Map<String,ContentStreamLoader> createDefaultLoaders(NamedList args) {
+    SolrParams p = null;
+    if(args!=null) {
+      p = SolrParams.toSolrParams(args);
+    }
+    Map<String,ContentStreamLoader> registry = new HashMap<String,ContentStreamLoader>();
+    registry.put("application/xml", new XMLLoader().init(p) );
+    registry.put("application/json", new JsonLoader().init(p) );
+    registry.put("application/csv", new CSVLoader().init(p) );
+    registry.put("application/javabin", new JavabinLoader().init(p) );
+    registry.put("text/csv", registry.get("application/csv") );
+    registry.put("text/xml", registry.get("application/xml") );
+    registry.put("text/json", registry.get("application/json") );
+    return registry;
+  }
+  
+
+  @Override
+  protected ContentStreamLoader newLoader(SolrQueryRequest req, final UpdateRequestProcessor processor) {
+    return instance;
+  }
+  
+  //////////////////////// SolrInfoMBeans methods //////////////////////
+
+  @Override
+  public String getDescription() {
+    return "Add documents using XML (with XSLT), CSV, JSON, or javabin";
+  }
+
+  @Override
+  public String getSource() {
+    return "$URL$";
+  }
+}
+
+
+

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java Tue May  8 21:06:10 2012
@@ -17,67 +17,21 @@
 
 package org.apache.solr.handler;
 
-import javax.xml.stream.XMLInputFactory;
-
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Add documents to solr using the STAX XML parser.
+ * 
+ * use {@link UpdateRequestHandler}
  */
-public class XmlUpdateRequestHandler extends ContentStreamHandlerBase {
-  public static Logger log = LoggerFactory.getLogger(XmlUpdateRequestHandler.class);
-  private static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
-
-  // XML Constants
-  public static final String ADD = "add";
-  public static final String DELETE = "delete";
-  public static final String OPTIMIZE = "optimize";
-  public static final String COMMIT = "commit";
-  public static final String ROLLBACK = "rollback";
-  public static final String WAIT_SEARCHER = "waitSearcher";
-  public static final String SOFT_COMMIT = "softCommit";
-
-  public static final String OVERWRITE = "overwrite";
-  
-  public static final String VERSION = "version";
-  
-  // NOTE: This constant is for use with the <add> XML tag, not the HTTP param with same name
-  public static final String COMMIT_WITHIN = "commitWithin";
-
-
-  XMLInputFactory inputFactory;
-
+@Deprecated
+public class XmlUpdateRequestHandler extends UpdateRequestHandler {
 
   @Override
   public void init(NamedList args) {
     super.init(args);
-
-    inputFactory = XMLInputFactory.newInstance();
-    try {
-      // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
-      // XMLInputFactory, as that implementation tries to cache and reuse the
-      // XMLStreamReader.  Setting the parser-specific "reuse-instance" property to false
-      // prevents this.
-      // All other known open-source stax parsers (and the bea ref impl)
-      // have thread-safe factories.
-      inputFactory.setProperty("reuse-instance", Boolean.FALSE);
-    }
-    catch (IllegalArgumentException ex) {
-      // Other implementations will likely throw this exception since "reuse-instance"
-      // isimplementation specific.
-      log.debug("Unable to set the 'reuse-instance' property for the input chain: " + inputFactory);
-    }
-    inputFactory.setXMLReporter(xmllog);
-  }
-
-  @Override
-  protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
-    return new XMLLoader(processor, inputFactory);
+    setAssumeContentType("application/xml");
+    log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
   }
 
   //////////////////////// SolrInfoMBeans methods //////////////////////

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java?rev=1335768&r1=1335767&r2=1335768&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java Tue May  8 21:06:10 2012
@@ -17,58 +17,21 @@
 
 package org.apache.solr.handler;
 
-import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.XMLErrorLogger;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.xml.stream.XMLInputFactory;
 
 /**
  * Add documents to solr using the STAX XML parser, transforming it with XSLT first
+ * 
+ * use {@link UpdateRequestHandler}
  */
-public class XsltUpdateRequestHandler extends ContentStreamHandlerBase {
-  public static Logger log = LoggerFactory.getLogger(XsltUpdateRequestHandler.class);
-  public static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
-
-  public static final int XSLT_CACHE_DEFAULT = 60;
-  private static final String XSLT_CACHE_PARAM = "xsltCacheLifetimeSeconds"; 
-
-  XMLInputFactory inputFactory;
-  private Integer xsltCacheLifetimeSeconds;
+@Deprecated
+public class XsltUpdateRequestHandler extends UpdateRequestHandler {
 
   @Override
   public void init(NamedList args) {
     super.init(args);
-
-    inputFactory = XMLInputFactory.newInstance();
-    try {
-      // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
-      // XMLInputFactory, as that implementation tries to cache and reuse the
-      // XMLStreamReader.  Setting the parser-specific "reuse-instance" property to false
-      // prevents this.
-      // All other known open-source stax parsers (and the bea ref impl)
-      // have thread-safe factories.
-      inputFactory.setProperty("reuse-instance", Boolean.FALSE);
-    }
-    catch (IllegalArgumentException ex) {
-      // Other implementations will likely throw this exception since "reuse-instance"
-      // isimplementation specific.
-      log.debug("Unable to set the 'reuse-instance' property for the input chain: " + inputFactory);
-    }
-    inputFactory.setXMLReporter(xmllog);
-    
-    final SolrParams p = SolrParams.toSolrParams(args);
-    this.xsltCacheLifetimeSeconds = p.getInt(XSLT_CACHE_PARAM,XSLT_CACHE_DEFAULT);
-    log.info("xsltCacheLifetimeSeconds=" + xsltCacheLifetimeSeconds);
-  }
-
-  @Override
-  protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
-    return new XsltXMLLoader(processor, inputFactory, xsltCacheLifetimeSeconds);
+    setAssumeContentType("application/xml");
+    log.warn("Using deprecated class: "+this.getClass().getSimpleName()+" -- replace with UpdateRequestHandler");
   }
 
   //////////////////////// SolrInfoMBeans methods //////////////////////

Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java Tue May  8 21:06:10 2012
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.loader;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import java.io.*;
+
+public class CSVLoader extends ContentStreamLoader {
+  @Override
+  public void load(SolrQueryRequest req, SolrQueryResponse rsp,
+      ContentStream stream, UpdateRequestProcessor processor) throws Exception {
+    new SingleThreadedCSVLoader(req,processor).load(req, rsp, stream, processor);
+  }
+}
+
+class SingleThreadedCSVLoader extends CSVLoaderBase {
+  SingleThreadedCSVLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
+    super(req, processor);
+  }
+
+  @Override
+  void addDoc(int line, String[] vals) throws IOException {
+    templateAdd.clear();
+    SolrInputDocument doc = new SolrInputDocument();
+    doAdd(line, vals, doc, templateAdd);
+  }
+}

Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java Tue May  8 21:06:10 2012
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.loader;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.update.*;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.internal.csv.CSVStrategy;
+import org.apache.solr.internal.csv.CSVParser;
+import org.apache.commons.io.IOUtils;
+
+import java.util.regex.Pattern;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.io.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class CSVLoaderBase extends ContentStreamLoader {
+  public static final String SEPARATOR="separator";
+  public static final String FIELDNAMES="fieldnames";
+  public static final String HEADER="header";
+  public static final String SKIP="skip";
+  public static final String SKIPLINES="skipLines";
+  public static final String MAP="map";
+  public static final String TRIM="trim";
+  public static final String EMPTY="keepEmpty";
+  public static final String SPLIT="split";
+  public static final String ENCAPSULATOR="encapsulator";
+  public static final String ESCAPE="escape";
+  public static final String OVERWRITE="overwrite";
+  public static final String LITERALS_PREFIX = "literal.";
+
+  private static Pattern colonSplit = Pattern.compile(":");
+  private static Pattern commaSplit = Pattern.compile(",");
+  
+  public static Logger log = LoggerFactory.getLogger(CSVLoaderBase.class);
+
+  final IndexSchema schema;
+  final SolrParams params;
+  final CSVStrategy strategy;
+  final UpdateRequestProcessor processor;
+
+  // hashmap to save any literal fields and their values
+  HashMap <SchemaField, String> literals;
+  String[] fieldnames;
+  SchemaField[] fields;
+  CSVLoaderBase.FieldAdder[] adders;
+
+  int skipLines;    // number of lines to skip at start of file
+
+  final AddUpdateCommand templateAdd;
+
+
+
+  /** Add a field to a document unless it's zero length.
+   * The FieldAdder hierarchy handles all the complexity of
+   * further transforming or splitting field values to keep the
+   * main logic loop clean.  All implementations of add() must be
+   * MT-safe!
+   */
+  private class FieldAdder {
+    void add(SolrInputDocument doc, int line, int column, String val) {
+      if (val.length() > 0) {
+        doc.addField(fields[column].getName(),val,1.0f);
+      }
+    }
+  }
+
+  /** add zero length fields */
+  private class FieldAdderEmpty extends CSVLoaderBase.FieldAdder {
+    @Override
+    void add(SolrInputDocument doc, int line, int column, String val) {
+      doc.addField(fields[column].getName(),val,1.0f);
+    }
+  }
+
+  /** trim fields */
+  private class FieldTrimmer extends CSVLoaderBase.FieldAdder {
+    private final CSVLoaderBase.FieldAdder base;
+    FieldTrimmer(CSVLoaderBase.FieldAdder base) { this.base=base; }
+    @Override
+    void add(SolrInputDocument doc, int line, int column, String val) {
+      base.add(doc, line, column, val.trim());
+    }
+  }
+
+  /** map a single value.
+   * for just a couple of mappings, this is probably faster than
+   * using a HashMap.
+   */
+ private class FieldMapperSingle extends CSVLoaderBase.FieldAdder {
+   private final String from;
+   private final String to;
+   private final CSVLoaderBase.FieldAdder base;
+   FieldMapperSingle(String from, String to, CSVLoaderBase.FieldAdder base) {
+     this.from=from;
+     this.to=to;
+     this.base=base;
+   }
+    @Override
+    void add(SolrInputDocument doc, int line, int column, String val) {
+      if (from.equals(val)) val=to;
+      base.add(doc,line,column,val);
+    }
+ }
+
+  /** Split a single value into multiple values based on
+   * a CSVStrategy.
+   */
+  private class FieldSplitter extends CSVLoaderBase.FieldAdder {
+    private final CSVStrategy strategy;
+    private final CSVLoaderBase.FieldAdder base;
+    FieldSplitter(CSVStrategy strategy, CSVLoaderBase.FieldAdder base) {
+      this.strategy = strategy;
+      this.base = base;
+    }
+
+    @Override
+    void add(SolrInputDocument doc, int line, int column, String val) {
+      CSVParser parser = new CSVParser(new StringReader(val), strategy);
+      try {
+        String[] vals = parser.getLine();
+        if (vals!=null) {
+          for (String v: vals) base.add(doc,line,column,v);
+        } else {
+          base.add(doc,line,column,val);
+        }
+      } catch (IOException e) {
+        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,e);
+      }
+    }
+  }
+
+
+  String errHeader="CSVLoader:";
+
+  CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
+    this.processor = processor;
+    this.params = req.getParams();
+    schema = req.getSchema();
+    this.literals = new HashMap<SchemaField, String>();
+
+    templateAdd = new AddUpdateCommand(req);
+    templateAdd.overwrite=params.getBool(OVERWRITE,true);
+    templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
+    
+    strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
+    String sep = params.get(SEPARATOR);
+    if (sep!=null) {
+      if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
+      strategy.setDelimiter(sep.charAt(0));
+    }
+
+    String encapsulator = params.get(ENCAPSULATOR);
+    if (encapsulator!=null) {
+      if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
+    }
+
+    String escape = params.get(ESCAPE);
+    if (escape!=null) {
+      if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
+    }
+
+    // if only encapsulator or escape is set, disable the other escaping mechanism
+    if (encapsulator == null && escape != null) {
+      strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);     
+      strategy.setEscape(escape.charAt(0));
+    } else {
+      if (encapsulator != null) {
+        strategy.setEncapsulator(encapsulator.charAt(0));
+      }
+      if (escape != null) {
+        char ch = escape.charAt(0);
+        strategy.setEscape(ch);
+        if (ch == '\\') {
+          // If the escape is the standard backslash, then also enable
+          // unicode escapes (it's harmless since 'u' would not otherwise
+          // be escaped.                    
+          strategy.setUnicodeEscapeInterpretation(true);
+        }
+      }
+    }
+
+    String fn = params.get(FIELDNAMES);
+    fieldnames = fn != null ? commaSplit.split(fn,-1) : null;
+
+    Boolean hasHeader = params.getBool(HEADER);
+
+    skipLines = params.getInt(SKIPLINES,0);
+
+    if (fieldnames==null) {
+      if (null == hasHeader) {
+        // assume the file has the headers if they aren't supplied in the args
+        hasHeader=true;
+      } else if (!hasHeader) {
+        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"CSVLoader: must specify fieldnames=<fields>* or header=true");
+      }
+    } else {
+      // if the fieldnames were supplied and the file has a header, we need to
+      // skip over that header.
+      if (hasHeader!=null && hasHeader) skipLines++;
+
+      prepareFields();
+    }
+  }
+
+  /** create the FieldAdders that control how each field  is indexed */
+  void prepareFields() {
+    // Possible future optimization: for really rapid incremental indexing
+    // from a POST, one could cache all of this setup info based on the params.
+    // The link from FieldAdder to this would need to be severed for that to happen.
+
+    fields = new SchemaField[fieldnames.length];
+    adders = new CSVLoaderBase.FieldAdder[fieldnames.length];
+    String skipStr = params.get(SKIP);
+    List<String> skipFields = skipStr==null ? null : StrUtils.splitSmart(skipStr,',');
+
+    CSVLoaderBase.FieldAdder adder = new CSVLoaderBase.FieldAdder();
+    CSVLoaderBase.FieldAdder adderKeepEmpty = new CSVLoaderBase.FieldAdderEmpty();
+
+    for (int i=0; i<fields.length; i++) {
+      String fname = fieldnames[i];
+      // to skip a field, leave the entries in fields and addrs null
+      if (fname.length()==0 || (skipFields!=null && skipFields.contains(fname))) continue;
+
+      fields[i] = schema.getField(fname);
+      boolean keepEmpty = params.getFieldBool(fname,EMPTY,false);
+      adders[i] = keepEmpty ? adderKeepEmpty : adder;
+
+      // Order that operations are applied: split -> trim -> map -> add
+      // so create in reverse order.
+      // Creation of FieldAdders could be optimized and shared among fields
+
+      String[] fmap = params.getFieldParams(fname,MAP);
+      if (fmap!=null) {
+        for (String mapRule : fmap) {
+          String[] mapArgs = colonSplit.split(mapRule,-1);
+          if (mapArgs.length!=2)
+            throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Map rules must be of the form 'from:to' ,got '"+mapRule+"'");
+          adders[i] = new CSVLoaderBase.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
+        }
+      }
+ 
+      if (params.getFieldBool(fname,TRIM,false)) {
+        adders[i] = new CSVLoaderBase.FieldTrimmer(adders[i]);
+      }
+
+      if (params.getFieldBool(fname,SPLIT,false)) {
+        String sepStr = params.getFieldParam(fname,SEPARATOR);
+        char fsep = sepStr==null || sepStr.length()==0 ? ',' : sepStr.charAt(0);
+        String encStr = params.getFieldParam(fname,ENCAPSULATOR);
+        char fenc = encStr==null || encStr.length()==0 ? (char)-2 : encStr.charAt(0);
+        String escStr = params.getFieldParam(fname,ESCAPE);
+        char fesc = escStr==null || escStr.length()==0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0);
+
+        CSVStrategy fstrat = new CSVStrategy(fsep,fenc,CSVStrategy.COMMENTS_DISABLED,fesc, false, false, false, false);
+        adders[i] = new CSVLoaderBase.FieldSplitter(fstrat, adders[i]);
+      }
+    }
+    // look for any literal fields - literal.foo=xyzzy
+    Iterator<String> paramNames = params.getParameterNamesIterator();
+    while (paramNames.hasNext()) {
+      String pname = paramNames.next();
+      if (!pname.startsWith(LITERALS_PREFIX)) continue;
+
+      String name = pname.substring(LITERALS_PREFIX.length());
+      SchemaField sf = schema.getFieldOrNull(name);
+      if(sf == null)
+        throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
+      literals.put(sf, params.get(pname));
+    }
+  }
+
+  private void input_err(String msg, String[] line, int lineno) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(errHeader).append(", line=").append(lineno).append(",").append(msg).append("\n\tvalues={");
+    for (String val: line) {
+      sb.append("'").append(val).append("',"); }
+    sb.append('}');
+    throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString());
+  }
+
+  private void input_err(String msg, String[] lines, int lineNo, Throwable e) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(errHeader).append(", line=").append(lineNo).append(",").append(msg).append("\n\tvalues={");
+    if (lines != null) {
+      for (String val : lines) {
+        sb.append("'").append(val).append("',");
+      }
+    } else {
+      sb.append("NO LINES AVAILABLE");
+    }
+    sb.append('}');
+    throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,sb.toString(), e);
+  }
+
+  /** load the CSV input */
+  @Override
+  public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws IOException {
+    errHeader = "CSVLoader: input=" + stream.getSourceInfo();
+    Reader reader = null;
+    try {
+      reader = stream.getReader();
+      if (skipLines>0) {
+        if (!(reader instanceof BufferedReader)) {
+          reader = new BufferedReader(reader);
+        }
+        BufferedReader r = (BufferedReader)reader;
+        for (int i=0; i<skipLines; i++) {
+          r.readLine();
+        }
+      }
+
+      CSVParser parser = new CSVParser(reader, strategy);
+
+      // parse the fieldnames from the header of the file
+      if (fieldnames==null) {
+        fieldnames = parser.getLine();
+        if (fieldnames==null) {
+          throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
+        }
+        prepareFields();
+      }
+
+      // read the rest of the CSV file
+      for(;;) {
+        int line = parser.getLineNumber();  // for error reporting in MT mode
+        String[] vals = null;
+        try {
+          vals = parser.getLine();
+        } catch (IOException e) {
+          //Catch the exception and rethrow it with more line information
+         input_err("can't read line: " + line, null, line, e);
+        }
+        if (vals==null) break;
+
+        if (vals.length != fields.length) {
+          input_err("expected "+fields.length+" values but got "+vals.length, vals, line);
+        }
+
+        addDoc(line,vals);
+      }
+    } finally{
+      if (reader != null) {
+        IOUtils.closeQuietly(reader);
+      }
+    }
+  }
+
+  /** called for each line of values (document) */
+  abstract void addDoc(int line, String[] vals) throws IOException;
+
+  /** this must be MT safe... may be called concurrently from multiple threads. */
+  void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template) throws IOException {
+    // the line number is passed simply for error reporting in MT mode.
+    // first, create the lucene document
+    for (int i=0; i<vals.length; i++) {
+      if (fields[i]==null) continue;  // ignore this field
+      String val = vals[i];
+      adders[i].add(doc, line, i, val);
+    }
+
+    // add any literals
+    for (SchemaField sf : literals.keySet()) {
+    	String fn = sf.getName();
+    	String val = literals.get(sf);
+    	doc.addField(fn, val);
+    }
+   
+    template.solrDoc = doc;
+    processor.processAdd(template);
+  }
+}
\ No newline at end of file

Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java?rev=1335768&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/loader/ContentStreamLoader.java Tue May  8 21:06:10 2012
@@ -0,0 +1,57 @@
+package org.apache.solr.handler.loader;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+
+/**
+ * Load a {@link org.apache.solr.common.util.ContentStream} into Solr
+ *
+ * This should be thread safe and can be called from multiple threads
+ */
+public abstract class ContentStreamLoader {
+
+  /**
+   * This should be called once for each RequestHandler
+   */
+  public ContentStreamLoader init(SolrParams args) {
+    return this;
+  }
+  
+  public String getDefaultWT() {
+    return null;
+  }
+
+  /**
+   * Loaders are responsible for closing the stream
+   *
+   * @param req The input {@link org.apache.solr.request.SolrQueryRequest}
+   * @param rsp The response, in case the Loader wishes to add anything
+   * @param stream The {@link org.apache.solr.common.util.ContentStream} to add
+   * @param processor The {@link UpdateRequestProcessor} to use
+   */
+  public abstract void load(SolrQueryRequest req, 
+      SolrQueryResponse rsp, 
+      ContentStream stream, 
+      UpdateRequestProcessor processor) throws Exception;
+}