You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2013/07/30 01:24:22 UTC

svn commit: r1508236 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/java/org/apache/solr/servlet/SolrRequestParsers.java core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java

Author: uschindler
Date: Mon Jul 29 23:24:22 2013
New Revision: 1508236

URL: http://svn.apache.org/r1508236
Log:
SOLR-5082: The encoding of URL-encoded query parameters can be changed with the "ie" (input encoding) parameter, e.g. "select?q=m%FCller&ie=ISO-8859-1". The default is UTF-8. To change the encoding of POSTed content, use the "Content-Type" HTTP header

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1508236&r1=1508235&r2=1508236&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Mon Jul 29 23:24:22 2013
@@ -71,6 +71,11 @@ New Features
   field is referenced via 'sfield' and the query point is constant.
   (David Smiley)
 
+* SOLR-5082: The encoding of URL-encoded query parameters can be changed with
+  the "ie" (input encoding) parameter, e.g. "select?q=m%FCller&ie=ISO-8859-1".
+  The default is UTF-8. To change the encoding of POSTed content, use the
+  "Content-Type" HTTP header.  (Uwe Schindler, David Smiley)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java?rev=1508236&r1=1508235&r2=1508236&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java Mon Jul 29 23:24:22 2013
@@ -28,9 +28,11 @@ import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CodingErrorAction;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -68,6 +70,11 @@ public class SolrRequestParsers 
   public static final String SIMPLE = "simple";
   public static final String STANDARD = "standard";
   
+  private static final Charset CHARSET_US_ASCII = Charset.forName("US-ASCII");
+  
+  public static final String INPUT_ENCODING_KEY = "ie";
+  private static final byte[] INPUT_ENCODING_BYTES = INPUT_ENCODING_KEY.getBytes(CHARSET_US_ASCII);
+
   private final HashMap<String, SolrRequestParser> parsers =
       new HashMap<String, SolrRequestParser>();
   private final boolean enableRemoteStreams;
@@ -242,7 +249,7 @@ public class SolrRequestParsers 
             }
           }
         };
-        parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map);
+        parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map, true);
       } catch (IOException ioe) {
         throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
       }
@@ -256,23 +263,53 @@ public class SolrRequestParsers 
    * @param charset to be used to decode resulting bytes after %-decoding
    * @param map place all parameters in this map
    */
-  @SuppressWarnings("fallthrough")
-  static long parseFormDataContent(final InputStream postContent, final long maxLen, final Charset charset, final Map<String,String[]> map) throws IOException {
-    final CharsetDecoder charsetDecoder = charset.newDecoder()
-      .onMalformedInput(CodingErrorAction.REPORT)
-      .onUnmappableCharacter(CodingErrorAction.REPORT);
+  @SuppressWarnings({"fallthrough", "resource"})
+  static long parseFormDataContent(final InputStream postContent, final long maxLen, Charset charset, final Map<String,String[]> map, boolean supportCharsetParam) throws IOException {
+    CharsetDecoder charsetDecoder = supportCharsetParam ? null : getCharsetDecoder(charset);
+    final LinkedList<Object> buffer = supportCharsetParam ? new LinkedList<Object>() : null;
     long len = 0L, keyPos = 0L, valuePos = 0L;
-    final ByteArrayOutputStream2 keyStream = new ByteArrayOutputStream2(),
-      valueStream = new ByteArrayOutputStream2();
-    ByteArrayOutputStream2 currentStream = keyStream;
+    final ByteArrayOutputStream keyStream = new ByteArrayOutputStream(),
+      valueStream = new ByteArrayOutputStream();
+    ByteArrayOutputStream currentStream = keyStream;
     for(;;) {
       int b = postContent.read();
       switch (b) {
         case -1: // end of stream
         case '&': // separator
           if (keyStream.size() > 0) {
-            final String key = decodeChars(keyStream, keyPos, charsetDecoder), value = decodeChars(valueStream, valuePos, charsetDecoder);
-            MultiMapSolrParams.addParam(key, value, map);
+            final byte[] keyBytes = keyStream.toByteArray(), valueBytes = valueStream.toByteArray();
+            if (Arrays.equals(keyBytes, INPUT_ENCODING_BYTES)) {
+              // we found a charset declaration in the raw bytes
+              if (charsetDecoder != null) {
+                throw new SolrException(ErrorCode.BAD_REQUEST,
+                  supportCharsetParam ? (
+                    "Query string invalid: duplicate '"+
+                    INPUT_ENCODING_KEY + "' (input encoding) key."
+                  ) : (
+                    "Key '" + INPUT_ENCODING_KEY + "' (input encoding) cannot "+
+                    "be used in POSTed application/x-www-form-urlencoded form data. "+
+                    "To set the input encoding of POSTed form data, use the "+
+                    "'Content-Type' header and provide a charset!"
+                  )
+                );
+              }
+              // decode the charset from raw bytes
+              charset = Charset.forName(decodeChars(valueBytes, keyPos, getCharsetDecoder(CHARSET_US_ASCII)));
+              charsetDecoder = getCharsetDecoder(charset);
+              // finally decode all buffered tokens
+              decodeBuffer(buffer, map, charsetDecoder);
+            } else if (charsetDecoder == null) {
+              // we have no charset decoder until now, buffer the keys / values for later processing:
+              buffer.add(keyBytes);
+              buffer.add(Long.valueOf(keyPos));
+              buffer.add(valueBytes);
+              buffer.add(Long.valueOf(valuePos));
+            } else {
+              // we already have a charsetDecoder, so we can directly decode without buffering:
+              final String key = decodeChars(keyBytes, keyPos, charsetDecoder),
+                  value = decodeChars(valueBytes, valuePos, charsetDecoder);
+              MultiMapSolrParams.addParam(key, value, map);
+            }
           } else if (valueStream.size() > 0) {
             throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded invalid: missing key");
           }
@@ -309,12 +346,23 @@ public class SolrRequestParsers 
         throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content exceeds upload limit of " + (maxLen/1024L) + " KB");
       }
     }
+    // if we have not seen a charset declaration, decode the buffer now using the default one (UTF-8 or given via Content-Type):
+    if (buffer != null && !buffer.isEmpty()) {
+      assert charsetDecoder == null;
+      decodeBuffer(buffer, map, getCharsetDecoder(charset));
+    }
     return len;
   }
   
-  private static String decodeChars(ByteArrayOutputStream2 stream, long position, CharsetDecoder charsetDecoder) {
+  private static CharsetDecoder getCharsetDecoder(Charset charset) {
+    return charset.newDecoder()
+      .onMalformedInput(CodingErrorAction.REPORT)
+      .onUnmappableCharacter(CodingErrorAction.REPORT);
+  }
+  
+  private static String decodeChars(byte[] bytes, long position, CharsetDecoder charsetDecoder) {
     try {
-      return charsetDecoder.decode(ByteBuffer.wrap(stream.buffer(), 0, stream.size())).toString();
+      return charsetDecoder.decode(ByteBuffer.wrap(bytes)).toString();
     } catch (CharacterCodingException cce) {
       throw new SolrException(ErrorCode.BAD_REQUEST,
         "URLDecoder: Invalid character encoding detected after position " + position +
@@ -323,10 +371,18 @@ public class SolrRequestParsers 
     }
   }
   
-  /** Makes the buffer of ByteArrayOutputStream available without copy. */
-  static final class ByteArrayOutputStream2 extends ByteArrayOutputStream {
-    byte[] buffer() {
-      return buf;
+  private static void decodeBuffer(final LinkedList<Object> input, final Map<String,String[]> map, CharsetDecoder charsetDecoder) {
+    for (final Iterator<Object> it = input.iterator(); it.hasNext(); ) {
+      final byte[] keyBytes = (byte[]) it.next();
+      it.remove();
+      final Long keyPos = (Long) it.next();
+      it.remove();
+      final byte[] valueBytes = (byte[]) it.next();
+      it.remove();
+      final Long valuePos = (Long) it.next();
+      it.remove();
+      MultiMapSolrParams.addParam(decodeChars(keyBytes, keyPos.longValue(), charsetDecoder),
+          decodeChars(valueBytes, valuePos.longValue(), charsetDecoder), map);
     }
   }
   
@@ -551,7 +607,7 @@ public class SolrRequestParsers 
       InputStream in = null;
       try {
         in = req.getInputStream();
-        final long bytesRead = parseFormDataContent(FastInputStream.wrap(in), maxLength, charset, map);
+        final long bytesRead = parseFormDataContent(FastInputStream.wrap(in), maxLength, charset, map, false);
         if (bytesRead == 0L && totalLength > 0L) {
           throw getParameterIncompatibilityException();
         }

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java?rev=1508236&r1=1508235&r2=1508236&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java Mon Jul 29 23:24:22 2013
@@ -243,6 +243,38 @@ public class SolrRequestParserTest exten
   }
   
   @Test
+  public void testStandardParseParamsAndFillStreamsISO88591() throws Exception
+  {
+    final String getParams = "qt=%FC&dup=foo&ie=iso-8859-1&dup=%FC", postParams = "qt2=%FC&q=hello&d%75p=bar";
+    final byte[] postBytes = postParams.getBytes("US-ASCII");
+    final String contentType = "application/x-www-form-urlencoded; charset=iso-8859-1";
+    
+    // Set up the expected behavior
+    HttpServletRequest request = createMock(HttpServletRequest.class);
+    expect(request.getMethod()).andReturn("POST").anyTimes();
+    expect(request.getContentType()).andReturn( contentType ).anyTimes();
+    expect(request.getQueryString()).andReturn(getParams).anyTimes();
+    expect(request.getContentLength()).andReturn(postBytes.length).anyTimes();
+    expect(request.getInputStream()).andReturn(new ServletInputStream() {
+      private final ByteArrayInputStream in = new ByteArrayInputStream(postBytes);
+      @Override public int read() { return in.read(); }
+    });
+    replay(request);
+    
+    MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
+    RawRequestParser raw = new RawRequestParser();
+    FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
+    StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
+    
+    SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
+    
+    assertEquals( "contentType: "+contentType, "hello", p.get("q") );
+    assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
+    assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt2") );
+    assertArrayEquals( "contentType: "+contentType, new String[]{"foo","\u00FC","bar"}, p.getParams("dup") );
+  }
+  
+  @Test
   public void testStandardFormdataUploadLimit() throws Exception
   {
     final int limitKBytes = 128;