You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/02/04 13:01:49 UTC
svn commit: r1067160 - in /lucene/dev/trunk/solr:
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/
src/java/org/apache/solr/core/ src/java/org/apache/solr/handler/
src/java/org/apache/solr/handler/admin/ src/test/org/apache/s...
Author: uschindler
Date: Fri Feb 4 12:01:49 2011
New Revision: 1067160
URL: http://svn.apache.org/viewvc?rev=1067160&view=rev
Log:
Fix lots of default charset violations in Solr caused by String.getBytes() and IOUtils.toString() [and others]
Modified:
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
lucene/dev/trunk/solr/src/java/org/apache/solr/core/CoreContainer.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DumpRequestHandler.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java
lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java
lucene/dev/trunk/solr/src/test/org/apache/solr/cloud/ZkTestServer.java
lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/ContentStreamTest.java
lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/DOMUtilTest.java
lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
lucene/dev/trunk/solr/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java Fri Feb 4 12:01:49 2011
@@ -19,9 +19,10 @@ package org.apache.solr.handler.dataimpo
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.ByteArrayInputStream;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
@@ -55,7 +56,7 @@ public class TestDataConfig extends Abst
public void testBasic() throws Exception {
javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory
.newInstance().newDocumentBuilder();
- Document doc = builder.parse(new ByteArrayInputStream(xml.getBytes()));
+ Document doc = builder.parse(new InputSource(new StringReader(xml)));
DataConfig dc = new DataConfig();
dc.readFromXml(doc.getDocumentElement());
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/core/CoreContainer.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/core/CoreContainer.java Fri Feb 4 12:01:49 2011
@@ -232,7 +232,7 @@ public class CoreContainer
cores.load(solrHome, fconf);
} else {
log.info("no solr.xml file found - using default");
- cores.load(solrHome, new ByteArrayInputStream(DEF_SOLR_XML.getBytes()));
+ cores.load(solrHome, new ByteArrayInputStream(DEF_SOLR_XML.getBytes("UTF-8")));
cores.configFile = fconf;
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DumpRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DumpRequestHandler.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DumpRequestHandler.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DumpRequestHandler.java Fri Feb 4 12:01:49 2011
@@ -18,7 +18,7 @@
package org.apache.solr.handler;
import java.io.IOException;
-import java.io.InputStream;
+import java.io.Reader;
import java.util.ArrayList;
import org.apache.commons.io.IOUtils;
@@ -46,11 +46,11 @@ public class DumpRequestHandler extends
stream.add( "sourceInfo", content.getSourceInfo() );
stream.add( "size", content.getSize() );
stream.add( "contentType", content.getContentType() );
- InputStream is = content.getStream();
+ Reader reader = content.getReader();
try {
- stream.add( "stream", IOUtils.toString(is) );
+ stream.add( "stream", IOUtils.toString(reader) );
} finally {
- is.close();
+ reader.close();
}
streams.add( stream );
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java Fri Feb 4 12:01:49 2011
@@ -210,7 +210,7 @@ public class ShowFileRequestHandler exte
}
try {
InputStream input = core.getResourceLoader().openResource(path);
- return IOUtils.toString( input );
+ return IOUtils.toString( input, "UTF-8" );
}
catch( Exception ex ) {} // ignore it
return "";
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java Fri Feb 4 12:01:49 2011
@@ -185,6 +185,7 @@ public class SystemInfoHandler extends R
try {
Process process = Runtime.getRuntime().exec(cmd);
in = new DataInputStream( process.getInputStream() );
+ // use default charset from locale here, because the command invoked also uses the default locale:
return IOUtils.toString( in );
}
catch( Exception ex ) {
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java Fri Feb 4 12:01:49 2011
@@ -86,23 +86,23 @@ public class JettyWebappTest extends Luc
// sure they compile ok
String adminPath = "http://localhost:"+port+context+"/";
- String html = IOUtils.toString( new URL(adminPath).openStream() );
- assertNotNull( html ); // real error will be an exception
+ byte[] bytes = IOUtils.toByteArray( new URL(adminPath).openStream() );
+ assertNotNull( bytes ); // real error will be an exception
adminPath += "admin/";
- html = IOUtils.toString( new URL(adminPath).openStream() );
- assertNotNull( html ); // real error will be an exception
+ bytes = IOUtils.toByteArray( new URL(adminPath).openStream() );
+ assertNotNull( bytes ); // real error will be an exception
// analysis
- html = IOUtils.toString( new URL(adminPath+"analysis.jsp").openStream() );
- assertNotNull( html ); // real error will be an exception
+ bytes = IOUtils.toByteArray( new URL(adminPath+"analysis.jsp").openStream() );
+ assertNotNull( bytes ); // real error will be an exception
// schema browser
- html = IOUtils.toString( new URL(adminPath+"schema.jsp").openStream() );
- assertNotNull( html ); // real error will be an exception
+ bytes = IOUtils.toByteArray( new URL(adminPath+"schema.jsp").openStream() );
+ assertNotNull( bytes ); // real error will be an exception
// schema browser
- html = IOUtils.toString( new URL(adminPath+"threaddump.jsp").openStream() );
- assertNotNull( html ); // real error will be an exception
+ bytes = IOUtils.toByteArray( new URL(adminPath+"threaddump.jsp").openStream() );
+ assertNotNull( bytes ); // real error will be an exception
}
}
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/cloud/ZkTestServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/cloud/ZkTestServer.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/cloud/ZkTestServer.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/cloud/ZkTestServer.java Fri Feb 4 12:01:49 2011
@@ -279,7 +279,7 @@ public class ZkTestServer {
BufferedReader reader = null;
try {
OutputStream outstream = sock.getOutputStream();
- outstream.write(cmd.getBytes());
+ outstream.write(cmd.getBytes("US-ASCII"));
outstream.flush();
// this replicates NC - close the output stream before reading
sock.shutdownOutput();
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/ContentStreamTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/ContentStreamTest.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/ContentStreamTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/ContentStreamTest.java Fri Feb 4 12:01:49 2011
@@ -27,6 +27,7 @@ import java.io.InputStream;
import java.io.StringReader;
import java.net.ConnectException;
import java.net.URL;
+import java.net.URLConnection;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -64,12 +65,15 @@ public class ContentStreamTest extends L
public void testURLStream() throws IOException
{
- String content = null;
+ byte[] content = null;
+ String contentType = null;
URL url = new URL( "http://svn.apache.org/repos/asf/lucene/dev/trunk/" );
InputStream in = null;
try {
- in = url.openStream();
- content = IOUtils.toString( in );
+ URLConnection conn = url.openConnection();
+ in = conn.getInputStream();
+ contentType = conn.getContentType();
+ content = IOUtils.toByteArray(in);
} catch (ConnectException ex) {
assumeNoException("Unable to connect to " + url + " to run the test.", ex);
}finally {
@@ -78,23 +82,26 @@ public class ContentStreamTest extends L
}
}
- assertTrue( content.length() > 10 ); // found something...
+ assertTrue( content.length > 10 ); // found something...
ContentStreamBase stream = new ContentStreamBase.URLStream( url );
- assertEquals( content.length(), stream.getSize().intValue() );
+ assertEquals( content.length, stream.getSize().intValue() );
// Test the stream
in = stream.getStream();
try {
assertTrue( IOUtils.contentEquals(
- new ByteArrayInputStream( content.getBytes() ), in ) );
+ new ByteArrayInputStream(content), in ) );
}
finally {
IOUtils.closeQuietly(in);
}
+ String charset = ContentStreamBase.getCharsetFromContentType(contentType);
+ if (charset == null)
+ charset = ContentStreamBase.DEFAULT_CHARSET;
// Re-open the stream and this time use a reader
stream = new ContentStreamBase.URLStream( url );
- assertTrue( IOUtils.contentEquals( new StringReader( content ), stream.getReader() ) );
+ assertTrue( IOUtils.contentEquals( new StringReader(new String(content, charset)), stream.getReader() ) );
}
}
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/DOMUtilTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/DOMUtilTest.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/DOMUtilTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/common/util/DOMUtilTest.java Fri Feb 4 12:01:49 2011
@@ -17,7 +17,7 @@
package org.apache.solr.common.util;
-import java.io.ByteArrayInputStream;
+import java.io.StringReader;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -27,6 +27,7 @@ import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
+import org.xml.sax.InputSource;
import org.apache.lucene.util.LuceneTestCase;
@@ -85,6 +86,6 @@ public class DOMUtilTest extends LuceneT
}
public Document getDocument( String xml ) throws Exception {
- return builder.parse( new ByteArrayInputStream( xml.getBytes() ) );
+ return builder.parse(new InputSource(new StringReader(xml)));
}
}
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java Fri Feb 4 12:01:49 2011
@@ -177,7 +177,7 @@ public class DocumentAnalysisRequestHand
"</docs>"
).getBytes("ISO-8859-1");
- // we declare a content stream without charset:
+ // we declare a content stream with charset:
final ContentStream cs = new ByteStream(xmlBytes, "application/xml; charset=ISO-8859-1");
ModifiableSolrParams params = new ModifiableSolrParams();
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/servlet/SolrRequestParserTest.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/servlet/SolrRequestParserTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/servlet/SolrRequestParserTest.java Fri Feb 4 12:01:49 2011
@@ -74,7 +74,7 @@ public class SolrRequestParserTest exten
List<ContentStream> streams = new ArrayList<ContentStream>();
SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 1, streams.size() );
- assertEquals( body1, IOUtils.toString( streams.get(0).getStream() ) );
+ assertEquals( body1, IOUtils.toString( streams.get(0).getReader() ) );
req.close();
// Now add three and make sure they come out ok
@@ -87,9 +87,9 @@ public class SolrRequestParserTest exten
input.add( body1 );
input.add( body2 );
input.add( body3 );
- output.add( IOUtils.toString( streams.get(0).getStream() ) );
- output.add( IOUtils.toString( streams.get(1).getStream() ) );
- output.add( IOUtils.toString( streams.get(2).getStream() ) );
+ output.add( IOUtils.toString( streams.get(0).getReader() ) );
+ output.add( IOUtils.toString( streams.get(1).getReader() ) );
+ output.add( IOUtils.toString( streams.get(2).getReader() ) );
// sort them so the output is consistent
Collections.sort( input );
Collections.sort( output );
@@ -112,13 +112,13 @@ public class SolrRequestParserTest exten
{
boolean ok = false;
String url = "http://www.apache.org/dist/lucene/solr/";
- String txt = null;
+ byte[] bytes = null;
try {
URLConnection connection = new URL(url).openConnection();
connection.setConnectTimeout(5000);
connection.setReadTimeout(5000);
connection.connect();
- txt = IOUtils.toString( connection.getInputStream());
+ bytes = IOUtils.toByteArray( connection.getInputStream());
}
catch( Exception ex ) {
assumeNoException("Unable to connect to " + url + " to run the test.", ex);
@@ -134,7 +134,7 @@ public class SolrRequestParserTest exten
List<ContentStream> streams = new ArrayList<ContentStream>();
SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 1, streams.size() );
- assertEquals( txt, IOUtils.toString( streams.get(0).getStream() ) );
+ assertArrayEquals( bytes, IOUtils.toByteArray( streams.get(0).getStream() ) );
req.close();
}
Modified: lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1067160&r1=1067159&r2=1067160&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/trunk/solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java Fri Feb 4 12:01:49 2011
@@ -65,7 +65,7 @@ public class SolrDispatchFilter implemen
public SolrDispatchFilter() {
try {
- adminRequestParser = new SolrRequestParsers(new Config(null,"solr",new ByteArrayInputStream("<root/>".getBytes()),"") );
+ adminRequestParser = new SolrRequestParsers(new Config(null,"solr",new ByteArrayInputStream("<root/>".getBytes("UTF-8")),"") );
} catch (Exception e) {
//unlikely
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,e);