You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/03/09 20:36:55 UTC
svn commit: r1079955 - in /lucene/dev/branches/lucene_solr_3_1: ./ lucene/
solr/ solr/example/lib/ solr/src/java/org/apache/solr/handler/
solr/src/java/org/apache/solr/util/
solr/src/test/org/apache/solr/client/solrj/
solr/src/webapp/src/org/apache/sol...
Author: uschindler
Date: Wed Mar 9 19:36:54 2011
New Revision: 1079955
URL: http://svn.apache.org/viewvc?rev=1079955&view=rev
Log:
SOLR-2381: update jetty to a new patched version (fixes boundary issue); avoid slow/buggy jetty unicode conversion when possible; adds unicode test; update rest of deprecated Servlets to not use Writers/Readers from Servlet Container
Modified:
lucene/dev/branches/lucene_solr_3_1/ (props changed)
lucene/dev/branches/lucene_solr_3_1/lucene/ (props changed)
lucene/dev/branches/lucene_solr_3_1/solr/ (props changed)
lucene/dev/branches/lucene_solr_3_1/solr/CHANGES.txt
lucene/dev/branches/lucene_solr_3_1/solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar
lucene/dev/branches/lucene_solr_3_1/solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar
lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/util/TestHarness.java
lucene/dev/branches/lucene_solr_3_1/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrServlet.java
lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java
Modified: lucene/dev/branches/lucene_solr_3_1/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/CHANGES.txt?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/CHANGES.txt Wed Mar 9 19:36:54 2011
@@ -661,7 +661,7 @@ Other Changes
* SOLR-2381: Include a patched version of Jetty (6.1.26 + JETTY-1340)
to fix problematic UTF-8 handling for supplementary characters.
- (uschindler, yonik, rmuir)
+ (Bernd Fehling, uschindler, yonik, rmuir)
* SOLR-2391: The preferred Content-Type for XML was changed to
application/xml. XMLResponseWriter now only delivers using this
Modified: lucene/dev/branches/lucene_solr_3_1/solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene_solr_3_1/solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java Wed Mar 9 19:36:54 2011
@@ -18,6 +18,7 @@
package org.apache.solr.handler;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.XML;
import org.apache.solr.common.util.XMLErrorLogger;
@@ -32,7 +33,7 @@ import org.slf4j.LoggerFactory;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamReader;
-import java.io.Reader;
+import java.io.InputStream;
import java.io.Writer;
import java.util.HashMap;
@@ -111,25 +112,23 @@ public class XmlUpdateRequestHandler ext
* success or failure from an XML formated Update (from the Reader)
*
* @since solr 1.2
- * @deprecated Direct updates from a Reader, as well as the response
+ * @deprecated Direct updates from a Servlet, as well as the response
* format produced by this method, have been deprecated
* and will be removed in future versions. Any code using
* this method should be changed to use {@link #handleRequest}
* method with a ContentStream.
- * This metrhod is also broken regarding XML charset detection,
- * as XML files need to be parsed as InputStream and not as Reader.
*/
@Deprecated
- public void doLegacyUpdate(Reader input, Writer output) {
+ public void doLegacyUpdate(InputStream input, String inputContentType, Writer output) {
SolrCore core = SolrCore.getSolrCore();
SolrQueryRequest req = new LocalSolrQueryRequest(core, new HashMap<String,String[]>());
-
try {
// Old style requests do not choose a custom handler
UpdateRequestProcessorChain processorFactory = core.getUpdateProcessingChain(null);
-
SolrQueryResponse rsp = new SolrQueryResponse(); // ignored
- XMLStreamReader parser = inputFactory.createXMLStreamReader(input);
+ final String charset = ContentStreamBase.getCharsetFromContentType(inputContentType);
+ final XMLStreamReader parser = (charset == null) ?
+ inputFactory.createXMLStreamReader(input) : inputFactory.createXMLStreamReader(input, charset);
UpdateRequestProcessor processor = processorFactory.createProcessor(req, rsp);
XMLLoader loader = (XMLLoader) newLoader(req, processor);
loader.processUpdate(processor, parser);
Modified: lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/util/TestHarness.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/util/TestHarness.java?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/util/TestHarness.java (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/src/java/org/apache/solr/util/TestHarness.java Wed Mar 9 19:36:54 2011
@@ -44,7 +44,7 @@ import javax.xml.xpath.XPathFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.io.StringReader;
+import java.io.InputStream;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
@@ -200,12 +200,14 @@ public class TestHarness {
*/
@Deprecated
public String update(String xml) {
-
- StringReader req = new StringReader(xml);
- StringWriter writer = new StringWriter(32000);
-
- updater.doLegacyUpdate(req, writer);
- return writer.toString();
+ try {
+ final InputStream req = new ByteArrayInputStream(xml.getBytes("UTF-16BE"));
+ final StringWriter out = new StringWriter(32000);
+ updater.doLegacyUpdate(req, "application/xml; charset=UTF-16BE", out);
+ return out.toString();
+ } catch (UnsupportedEncodingException uue) {
+ throw new RuntimeException(uue); // cannot happen
+ }
}
Modified: lucene/dev/branches/lucene_solr_3_1/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java Wed Mar 9 19:36:54 2011
@@ -27,6 +27,10 @@ import java.util.List;
import junit.framework.Assert;
+import org.apache.lucene.util._TestUtil;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.DirectXmlRequest;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.request.SolrPing;
@@ -212,6 +216,90 @@ abstract public class SolrExampleTests e
}
+ private String randomTestString(int maxLength) {
+ // we can't just use _TestUtil.randomUnicodeString() or we might get 0xfffe etc
+ // (considered invalid by XML)
+
+ int size = random.nextInt(maxLength);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < size; i++) {
+ switch(random.nextInt(4)) {
+ case 0: /* single byte */
+ sb.append('a');
+ break;
+ case 1: /* two bytes */
+ sb.append('\u0645');
+ break;
+ case 2: /* three bytes */
+ sb.append('\u092a');
+ break;
+ case 3: /* four bytes */
+ sb.appendCodePoint(0x29B05);
+ }
+ }
+ return sb.toString();
+ }
+
+ public void testUnicode() throws Exception {
+ int numIterations = 100 * RANDOM_MULTIPLIER;
+
+ SolrServer server = getSolrServer();
+
+ // save the old parser, so we can set it back.
+ ResponseParser oldParser = null;
+ if (server instanceof CommonsHttpSolrServer) {
+ CommonsHttpSolrServer cserver = (CommonsHttpSolrServer) server;
+ oldParser = cserver.getParser();
+ }
+
+ try {
+ for (int iteration = 0; iteration < numIterations; iteration++) {
+ // choose format
+ if (server instanceof CommonsHttpSolrServer) {
+ if (random.nextBoolean()) {
+ ((CommonsHttpSolrServer) server).setParser(new BinaryResponseParser());
+ } else {
+ ((CommonsHttpSolrServer) server).setParser(new XMLResponseParser());
+ }
+ }
+
+ int numDocs = _TestUtil.nextInt(random, 1, 100);
+
+ // Empty the database...
+ server.deleteByQuery("*:*");// delete everything!
+
+ List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
+ for (int i = 0; i < numDocs; i++) {
+ // Now add something...
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "" + i);
+ doc.addField("unicode_s", randomTestString(30));
+ docs.add(doc);
+ }
+
+ server.add(docs);
+ server.commit();
+
+ SolrQuery query = new SolrQuery();
+ query.setQuery("*:*");
+ query.setRows(numDocs);
+
+ QueryResponse rsp = server.query( query );
+
+ for (int i = 0; i < numDocs; i++) {
+ String expected = (String) docs.get(i).getFieldValue("unicode_s");
+ String actual = (String) rsp.getResults().get(i).getFieldValue("unicode_s");
+ assertEquals(expected, actual);
+ }
+ }
+ } finally {
+ if (oldParser != null) {
+ // set the old parser back
+ ((CommonsHttpSolrServer)server).setParser(oldParser);
+ }
+ }
+ }
+
/**
* query the example
*/
Modified: lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java Wed Mar 9 19:36:54 2011
@@ -18,9 +18,12 @@
package org.apache.solr.servlet;
import java.io.IOException;
+import java.io.Writer;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.io.OutputStreamWriter;
import java.io.ByteArrayInputStream;
+import java.nio.charset.Charset;
import java.util.Map;
import java.util.WeakHashMap;
import org.slf4j.Logger;
@@ -40,6 +43,8 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.core.*;
import org.apache.solr.request.*;
import org.apache.solr.response.BinaryQueryResponseWriter;
@@ -63,6 +68,8 @@ public class SolrDispatchFilter implemen
protected String solrConfigFilename = null;
protected final Map<SolrConfig, SolrRequestParsers> parsers = new WeakHashMap<SolrConfig, SolrRequestParsers>();
protected final SolrRequestParsers adminRequestParser;
+
+ private static final Charset UTF8 = Charset.forName("UTF-8");
public SolrDispatchFilter() {
try {
@@ -322,14 +329,19 @@ public class SolrDispatchFilter implemen
final String ct = responseWriter.getContentType(solrReq, solrRsp);
// don't call setContentType on null
if (null != ct) response.setContentType(ct);
+
if (Method.HEAD != reqMethod) {
if (responseWriter instanceof BinaryQueryResponseWriter) {
BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) responseWriter;
binWriter.write(response.getOutputStream(), solrReq, solrRsp);
} else {
- PrintWriter out = response.getWriter();
+ String charset = ContentStreamBase.getCharsetFromContentType(ct);
+ Writer out = (charset == null || charset.equalsIgnoreCase("UTF-8"))
+ ? new OutputStreamWriter(response.getOutputStream(), UTF8)
+ : new OutputStreamWriter(response.getOutputStream(), charset);
+ out = new FastWriter(out);
responseWriter.write(out, solrReq, solrRsp);
-
+ out.flush();
}
}
//else http HEAD request, nothing to write out, waited this long just to get ContentType
Modified: lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrServlet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrServlet.java?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrServlet.java (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrServlet.java Wed Mar 9 19:36:54 2011
@@ -20,6 +20,9 @@ package org.apache.solr.servlet;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.io.Writer;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,10 +32,13 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.BinaryQueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
/**
@@ -48,6 +54,8 @@ public class SolrServlet extends HttpSer
final Logger log = LoggerFactory.getLogger(SolrServlet.class);
private boolean hasMulticore = false;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
@Override
public void init() throws ServletException {
log.info("SolrServlet.init()");
@@ -88,9 +96,22 @@ public class SolrServlet extends HttpSer
core.execute(handler, solrReq, solrRsp );
if (solrRsp.getException() == null) {
QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
- response.setContentType(responseWriter.getContentType(solrReq, solrRsp));
- PrintWriter out = response.getWriter();
- responseWriter.write(out, solrReq, solrRsp);
+ // Now write it out
+ final String ct = responseWriter.getContentType(solrReq, solrRsp);
+ // don't call setContentType on null
+ if (null != ct) response.setContentType(ct);
+ if (responseWriter instanceof BinaryQueryResponseWriter) {
+ BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) responseWriter;
+ binWriter.write(response.getOutputStream(), solrReq, solrRsp);
+ } else {
+ String charset = ContentStreamBase.getCharsetFromContentType(ct);
+ Writer out = (charset == null || charset.equalsIgnoreCase("UTF-8"))
+ ? new OutputStreamWriter(response.getOutputStream(), UTF8)
+ : new OutputStreamWriter(response.getOutputStream(), charset);
+ out = new FastWriter(out);
+ responseWriter.write(out, solrReq, solrRsp);
+ out.flush();
+ }
} else {
Exception e = solrRsp.getException();
int rc=500;
Modified: lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java?rev=1079955&r1=1079954&r2=1079955&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java (original)
+++ lucene/dev/branches/lucene_solr_3_1/solr/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java Wed Mar 9 19:36:54 2011
@@ -15,10 +15,12 @@ package org.apache.solr.servlet;/**
* limitations under the License.
*/
-import java.io.BufferedReader;
+import java.io.InputStream;
import java.io.File;
import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.Writer;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,6 +33,7 @@ import org.apache.solr.core.SolrResource
import org.apache.solr.handler.XmlUpdateRequestHandler;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.XMLResponseWriter;
+import org.apache.solr.common.util.FastWriter;
/**
* @version $Id$
@@ -45,6 +48,8 @@ public class SolrUpdateServlet extends H
XMLResponseWriter xmlResponseWriter;
private boolean hasMulticore = false;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
@Override
public void init() throws ServletException
{
@@ -66,7 +71,9 @@ public class SolrUpdateServlet extends H
response.sendError( 400, "Missing solr core name in path" );
return;
}
- BufferedReader requestReader = request.getReader();
+ final InputStream in = request.getInputStream();
+ final String inct = request.getContentType();
+
response.setContentType(QueryResponseWriter.CONTENT_TYPE_XML_UTF8);
if( request.getQueryString() != null ) {
@@ -75,7 +82,9 @@ public class SolrUpdateServlet extends H
+" If you are using solrj, make sure to register a request handler to /update rather then use this servlet.\n"
+" Add: <requestHandler name=\"/update\" class=\"solr.XmlUpdateRequestHandler\" > to your solrconfig.xml\n\n" );
}
- PrintWriter writer = response.getWriter();
- legacyUpdateHandler.doLegacyUpdate(requestReader, writer);
+
+ final Writer output = new FastWriter(new OutputStreamWriter(response.getOutputStream(), UTF8));
+ legacyUpdateHandler.doLegacyUpdate(in, inct, output);
+ output.flush();
}
}