You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/03/09 20:19:29 UTC
svn commit: r1079949 - in /lucene/dev/trunk/solr: ./ example/lib/
src/test/org/apache/solr/client/solrj/
src/webapp/src/org/apache/solr/servlet/
Author: rmuir
Date: Wed Mar 9 19:19:29 2011
New Revision: 1079949
URL: http://svn.apache.org/viewvc?rev=1079949&view=rev
Log:
SOLR-2381: update jetty to a new patched version (fixes boundary issue); avoid slow/buggy jetty unicode conversion when possible; adds unicode test
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar
lucene/dev/trunk/solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar
lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1079949&r1=1079948&r2=1079949&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Mar 9 19:19:29 2011
@@ -834,7 +834,7 @@ Other Changes
* SOLR-2381: Include a patched version of Jetty (6.1.26 + JETTY-1340)
to fix problematic UTF-8 handling for supplementary characters.
- (uschindler, yonik, rmuir)
+ (Bernd Fehling, uschindler, yonik, rmuir)
* SOLR-2391: The preferred Content-Type for XML was changed to
application/xml. XMLResponseWriter now only delivers using this
Modified: lucene/dev/trunk/solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/lib/jetty-6.1.26-patched-JETTY-1340.jar?rev=1079949&r1=1079948&r2=1079949&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/trunk/solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/lib/jetty-util-6.1.26-patched-JETTY-1340.jar?rev=1079949&r1=1079948&r2=1079949&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java?rev=1079949&r1=1079948&r2=1079949&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java Wed Mar 9 19:19:29 2011
@@ -27,6 +27,10 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import junit.framework.Assert;
+import org.apache.lucene.util._TestUtil;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.DirectXmlRequest;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.request.SolrPing;
@@ -215,6 +219,90 @@ abstract public class SolrExampleTests e
}
+ private String randomTestString(int maxLength) {
+ // we can't just use _TestUtil.randomUnicodeString() or we might get 0xfffe etc
+ // (considered invalid by XML)
+
+ int size = random.nextInt(maxLength);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < size; i++) {
+ switch(random.nextInt(4)) {
+ case 0: /* single byte */
+ sb.append('a');
+ break;
+ case 1: /* two bytes */
+ sb.append('\u0645');
+ break;
+ case 2: /* three bytes */
+ sb.append('\u092a');
+ break;
+ case 3: /* four bytes */
+ sb.appendCodePoint(0x29B05);
+ }
+ }
+ return sb.toString();
+ }
+
+ public void testUnicode() throws Exception {
+ int numIterations = 100 * RANDOM_MULTIPLIER;
+
+ SolrServer server = getSolrServer();
+
+ // save the old parser, so we can set it back.
+ ResponseParser oldParser = null;
+ if (server instanceof CommonsHttpSolrServer) {
+ CommonsHttpSolrServer cserver = (CommonsHttpSolrServer) server;
+ oldParser = cserver.getParser();
+ }
+
+ try {
+ for (int iteration = 0; iteration < numIterations; iteration++) {
+ // choose format
+ if (server instanceof CommonsHttpSolrServer) {
+ if (random.nextBoolean()) {
+ ((CommonsHttpSolrServer) server).setParser(new BinaryResponseParser());
+ } else {
+ ((CommonsHttpSolrServer) server).setParser(new XMLResponseParser());
+ }
+ }
+
+ int numDocs = _TestUtil.nextInt(random, 1, 100);
+
+ // Empty the database...
+ server.deleteByQuery("*:*");// delete everything!
+
+ List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
+ for (int i = 0; i < numDocs; i++) {
+ // Now add something...
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "" + i);
+ doc.addField("unicode_s", randomTestString(30));
+ docs.add(doc);
+ }
+
+ server.add(docs);
+ server.commit();
+
+ SolrQuery query = new SolrQuery();
+ query.setQuery("*:*");
+ query.setRows(numDocs);
+
+ QueryResponse rsp = server.query( query );
+
+ for (int i = 0; i < numDocs; i++) {
+ String expected = (String) docs.get(i).getFieldValue("unicode_s");
+ String actual = (String) rsp.getResults().get(i).getFieldValue("unicode_s");
+ assertEquals(expected, actual);
+ }
+ }
+ } finally {
+ if (oldParser != null) {
+ // set the old parser back
+ ((CommonsHttpSolrServer)server).setParser(oldParser);
+ }
+ }
+ }
+
/**
* query the example
*/
Modified: lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1079949&r1=1079948&r2=1079949&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java Wed Mar 9 19:19:29 2011
@@ -18,9 +18,12 @@
package org.apache.solr.servlet;
import java.io.IOException;
+import java.io.Writer;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.io.OutputStreamWriter;
import java.io.ByteArrayInputStream;
+import java.nio.charset.Charset;
import java.util.Map;
import java.util.WeakHashMap;
import org.slf4j.Logger;
@@ -40,6 +43,8 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.core.*;
import org.apache.solr.request.*;
import org.apache.solr.response.BinaryQueryResponseWriter;
@@ -63,6 +68,8 @@ public class SolrDispatchFilter implemen
protected String solrConfigFilename = null;
protected final Map<SolrConfig, SolrRequestParsers> parsers = new WeakHashMap<SolrConfig, SolrRequestParsers>();
protected final SolrRequestParsers adminRequestParser;
+
+ private static final Charset UTF8 = Charset.forName("UTF-8");
public SolrDispatchFilter() {
try {
@@ -319,14 +326,19 @@ public class SolrDispatchFilter implemen
final String ct = responseWriter.getContentType(solrReq, solrRsp);
// don't call setContentType on null
if (null != ct) response.setContentType(ct);
+
if (Method.HEAD != reqMethod) {
if (responseWriter instanceof BinaryQueryResponseWriter) {
BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) responseWriter;
binWriter.write(response.getOutputStream(), solrReq, solrRsp);
} else {
- PrintWriter out = response.getWriter();
+ String charset = ContentStreamBase.getCharsetFromContentType(ct);
+ Writer out = (charset == null || charset.equalsIgnoreCase("UTF-8"))
+ ? new OutputStreamWriter(response.getOutputStream(), UTF8)
+ : new OutputStreamWriter(response.getOutputStream(), charset);
+ out = new FastWriter(out);
responseWriter.write(out, solrReq, solrRsp);
-
+ out.flush();
}
}
//else http HEAD request, nothing to write out, waited this long just to get ContentType