You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@shindig.apache.org by jo...@apache.org on 2010/07/21 11:54:41 UTC
svn commit: r966159 - in /shindig/trunk/java/gadgets/src:
main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java
Author: johnh
Date: Wed Jul 21 09:54:41 2010
New Revision: 966159
URL: http://svn.apache.org/viewvc?rev=966159&view=rev
Log:
Workaround fix for JDK ZLIB bug reporting EOFException, provided by Vikas Arora.
Vikas' description:
Due to JDK bug (Ref:
http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4040920), the 'EOF Exception'
comes while accessing zip content. This issue is very inconsistent and happens
for some zip resources and that too once in 4-5 times.
This is little hard to reproduce but when it comes, it comes for wrong
case/reason, even though the resource itself is fetched fine. Refer the bug
link, mentioned above.
The bug lies in the JDK 'Inflater.finished()' call, that erroneously returns
'false' even if the input stream (zip) is done reading the response. Whenever
this happens, 'java.util.zip.InflaterInputStream.fill' will throw the
'EOFException'.
As a fix for this specific corner case, ignore the Exception (EOF with specific
cause trail) and let pass all other exceptions.
Verified this working OK on the site for which this error was initially
reported.
=====
Thanks Vikas!
Modified:
shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java
Modified: shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java
URL: http://svn.apache.org/viewvc/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java?rev=966159&r1=966158&r2=966159&view=diff
==============================================================================
--- shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java (original)
+++ shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/http/BasicHttpFetcher.java Wed Jul 21 09:54:41 2010
@@ -66,10 +66,12 @@ import org.apache.http.params.HttpConnec
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.ByteArrayBuffer;
import org.apache.http.util.EntityUtils;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.gadgets.GadgetException;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.net.ProxySelector;
@@ -464,11 +466,72 @@ public class BasicHttpFetcher implements
return HttpResponse.badrequest("Exceeded maximum number of bytes - " + maxObjSize);
}
- byte[] responseBytes = (entity == null) ? null : EntityUtils.toByteArray(entity);
+ byte[] responseBytes = (entity == null) ? null : toByteArraySafe(entity);
return builder
.setHttpStatusCode(response.getStatusLine().getStatusCode())
.setResponse(responseBytes)
.create();
}
+
+ /**
+ * This method is Safe replica version of org.apache.http.util.EntityUtils.toByteArray.
+ * The try block embedding 'instream.read' has a corresponding catch block for 'EOFException'
+ * (that's Ignored) and all other IOExceptions are let pass.
+ *
+ * @param entity
+ * @return byte array containing the entity content. May be empty/null.
+ * @throws IOException if an error occurs reading the input stream
+ */
+ public byte[] toByteArraySafe(final HttpEntity entity) throws IOException {
+ if (entity == null) {
+ return null;
+ }
+
+ InputStream instream = entity.getContent();
+ if (instream == null) {
+ return new byte[] {};
+ }
+ if (entity.getContentLength() > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("HTTP entity too large to be buffered in memory");
+ }
+
+ // The raw data stream (inside JDK) is read in a buffer of size '512'. The original code
+ // org.apache.http.util.EntityUtils.toByteArray reads the unzipped data in a buffer of
+ // 4096 byte. For any data stream that has a compression ratio lesser than 1/8, this may
+ // result in the buffer/array overflow. Increasing the buffer size to '16384'. It's highly
+ // unlikely to get data compression ratios lesser than 1/32 (3%).
+ final int bufferLength = 16384;
+ int i = (int)entity.getContentLength();
+ if (i < 0) {
+ i = bufferLength;
+ }
+ ByteArrayBuffer buffer = new ByteArrayBuffer(i);
+ try {
+ byte[] tmp = new byte[bufferLength];
+ int l;
+ while((l = instream.read(tmp)) != -1) {
+ buffer.append(tmp, 0, l);
+ }
+ } catch (EOFException eofe) {
+ // Ref: http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4040920
+ // Due to a bug in JDK ZLIB (InflaterInputStream), unexpected EOF error can occur.
+ // In such cases, even if the input stream is finished reading, the
+ // 'Inflater.finished()' call erroneously returns 'false' and
+ // 'java.util.zip.InflaterInputStream.fill' throws the 'EOFException'.
+ // So for such case, ignore the Exception in case Exception Cause is
+ // 'Unexpected end of ZLIB input stream'.
+ // For all other cases, re-throw the (EOF) Exception.
+ if ((instream.available() == 0) &&
+ eofe.getMessage().equals("Unexpected end of ZLIB input stream")) {
+ // Ignore
+ } else {
+ throw eofe;
+ }
+ }
+ finally {
+ instream.close();
+ }
+ return buffer.toByteArray();
+ }
}
Modified: shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java
URL: http://svn.apache.org/viewvc/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java?rev=966159&r1=966158&r2=966159&view=diff
==============================================================================
--- shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java (original)
+++ shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/http/BasicHttpFetcherTest.java Wed Jul 21 09:54:41 2010
@@ -17,22 +17,59 @@
*/
package org.apache.shindig.gadgets.http;
+import java.io.InputStream;
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.http.HttpEntity;
import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.common.uri.UriBuilder;
+import org.easymock.EasyMock;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Before;
import org.junit.Test;
-public class BasicHttpFetcherTest extends AbstractHttpFetcherTest {
+public class BasicHttpFetcherTest {
+ private static final int ECHO_PORT = 9003;
+ protected static final Uri BASE_URL = Uri.parse("http://localhost:9003/");
+ private static EchoServer server;
+
+ protected BasicHttpFetcher fetcher = null;
+ protected HttpEntity mockEntity;
+ protected InputStream mockInputStream;
+
+ @BeforeClass
+ public static void setUpOnce() throws Exception {
+ server = new EchoServer();
+ server.start(ECHO_PORT);
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ }
+
@Before
public void setUp() throws Exception {
- fetcher = new BasicHttpFetcher(null);
+ fetcher = new BasicHttpFetcher(BASE_URL.getAuthority());
+
+ mockInputStream = EasyMock.createMock(InputStream.class);
+ //EasyMock.expect(mockInputStream.available()).andReturn(0).anyTimes();
+ EasyMock.expect(mockInputStream.available()).andReturn(0);
+ mockInputStream.close();
+
+ mockEntity = EasyMock.createMock(HttpEntity.class);
+ EasyMock.expect(mockEntity.getContent()).andReturn(mockInputStream);
+ EasyMock.expect(mockEntity.getContentLength()).andReturn(new Long(16384)).anyTimes();
}
@Test
public void testWithProxy() throws Exception {
- fetcher = new BasicHttpFetcher(BASE_URL.getAuthority());
-
String content = "Hello, Gagan!";
Uri uri = new UriBuilder(Uri.parse("http://www.google.com/search"))
.addQueryParameter("body", content)
@@ -43,4 +80,66 @@ public class BasicHttpFetcherTest extend
assertEquals(201, response.getHttpStatusCode());
assertEquals(content, response.getResponseAsString());
}
+
+ @Test
+ public void testToByteArraySafeThrowsException1() throws Exception {
+ String exceptionMessage = "IO Exception and Any Random Cause";
+ IOException e = new IOException(exceptionMessage);
+ EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes();
+
+ EasyMock.replay(mockEntity, mockInputStream);
+
+ try {
+ fetcher.toByteArraySafe(mockEntity);
+ } catch (IOException ioe) {
+ assertEquals(exceptionMessage, ioe.getMessage());
+ }
+ }
+
+ @Test
+ public void testToByteArraySafeThrowsException2() throws Exception {
+ String exceptionMessage = "EOF Exception and Any Random Cause";
+ EOFException e = new EOFException(exceptionMessage);
+ EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes();
+
+ EasyMock.replay(mockEntity, mockInputStream);
+
+ try {
+ fetcher.toByteArraySafe(mockEntity);
+ } catch (EOFException eofe) {
+ assertEquals(exceptionMessage, eofe.getMessage());
+ }
+ }
+
+ @Test
+ public void testToByteArraySafeThrowsException3() throws Exception {
+ // Return non-zero for 'InputStream.available()'. This should violate the other condition.
+ EasyMock.expect(mockInputStream.available()).andReturn(1);
+ String exceptionMessage = "Unexpected end of ZLIB input stream";
+ EOFException e = new EOFException(exceptionMessage);
+ EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes();
+
+ EasyMock.replay(mockEntity, mockInputStream);
+
+ try {
+ fetcher.toByteArraySafe(mockEntity);
+ } catch (EOFException eofe) {
+ assertEquals(exceptionMessage, eofe.getMessage());
+ }
+ }
+
+ @Test
+ public void testToByteArraySafeHandleException() throws Exception {
+ String exceptionMessage = "Unexpected end of ZLIB input stream";
+ EOFException e = new EOFException(exceptionMessage);
+ EasyMock.expect(mockInputStream.read(EasyMock.isA(byte[].class))).andThrow(e).anyTimes();
+
+ EasyMock.replay(mockEntity, mockInputStream);
+
+ try {
+ fetcher.toByteArraySafe(mockEntity);
+ } catch (EOFException eofe) {
+ fail("Exception Should have been caught");
+ }
+ }
}