You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@shindig.apache.org by jo...@apache.org on 2010/01/12 01:51:08 UTC
svn commit: r898140 - in /incubator/shindig/trunk/java/gadgets/src:
main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
test/java/org/apache/shindig/gadgets/servlet/ConcatProxyServletTest.java
Author: johnh
Date: Tue Jan 12 00:51:08 2010
New Revision: 898140
URL: http://svn.apache.org/viewvc?rev=898140&view=rev
Log:
Server-side of split-JS support.
Introduces JSONP-esque parameter &json=<varname>, in which presence the /gadgets/concat servlet returns its data as a JSON object rather than a strictly concatenated set of JS, eg:
var _js={"url1":"escapedJs1", "url2":"escapedJs2"};
The varname is sanitized for security (must match [a-zA-Z0-9_]+). Rewriter-side support will try using this to reduce the number of HTTP requests for a given page with "split" JS blocks eg:
<script src="1.js">
<div>
foo
</div>
<script src="2.js">
turns into...
<script src="concat?1=1.js&2=2.js&json=_js">
<script>
eval(_js["1.js"]);
</script>
<div>
foo
<div>
<script>
eval(_js["2.js"]);
</script>
Added:
incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/ConcatProxyServletTest.java
Modified:
incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
Modified: incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java?rev=898140&r1=898139&r2=898140&view=diff
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java (original)
+++ incubator/shindig/trunk/java/gadgets/src/main/java/org/apache/shindig/gadgets/servlet/ConcatProxyServlet.java Tue Jan 12 00:51:08 2010
@@ -20,6 +20,7 @@
import com.google.inject.Inject;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.shindig.common.servlet.HttpUtil;
import org.apache.shindig.common.servlet.InjectedServlet;
import org.apache.shindig.gadgets.GadgetException;
@@ -31,7 +32,9 @@
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServletResponseWrapper;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.util.Locale;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -43,6 +46,8 @@
*/
public class ConcatProxyServlet extends InjectedServlet {
+ public static final String JSON_PARAM = "json";
+
private static final Logger logger
= Logger.getLogger(ConcatProxyServlet.class.getName());
@@ -67,7 +72,7 @@
response.setHeader("Content-Type",
request.getParameter(ProxyBase.REWRITE_MIME_TYPE_PARAM));
}
-
+
boolean ignoreCache = proxyHandler.getIgnoreCache(request);
if (!ignoreCache && request.getParameter(ProxyBase.REFRESH_PARAM) != null) {
HttpUtil.setCachingHeaders(response, Integer.valueOf(request
@@ -77,25 +82,37 @@
}
response.setHeader("Content-Disposition", "attachment;filename=p.txt");
+
+ // Check for json concat
+ String jsonVar = request.getParameter(JSON_PARAM);
+ if (jsonVar != null && !jsonVar.matches("^\\w*$")) {
+ response.getOutputStream().println(
+ formatHttpError(HttpServletResponse.SC_BAD_REQUEST,
+ "Bad json variable name " + jsonVar));
+ response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+
+ ResponseWrapper wrapper = new ResponseWrapper(response, jsonVar);
+
for (int i = 1; i < Integer.MAX_VALUE; i++) {
String url = request.getParameter(Integer.toString(i));
if (url == null) {
break;
}
try {
- response.getOutputStream().println("/* ---- Start " + url + " ---- */");
- ResponseWrapper wrapper = new ResponseWrapper(response);
+ wrapper.processUrl(url);
proxyHandler.doFetch(new RequestWrapper(request, url), wrapper);
if (wrapper.getStatus() != HttpServletResponse.SC_OK) {
response.getOutputStream().println(
formatHttpError(wrapper.getStatus(), wrapper.getErrorMessage()));
}
-
- response.getOutputStream().println("/* ---- End " + url + " ---- */");
+
} catch (GadgetException ge) {
if (ge.getCode() != GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT) {
+ wrapper.done();
outputError(ge, url, response);
return;
} else {
@@ -103,6 +120,7 @@
}
}
}
+ wrapper.done();
response.setStatus(200);
}
@@ -158,33 +176,89 @@
/**
* Wrap the response to prevent writing through of the status code and to hold a reference to the
* stream across multiple proxied parts
+ * Handles json concatenation by using the EscapedServletOutputStream class
+ * to escape the data
*/
private static class ResponseWrapper extends HttpServletResponseWrapper {
private ServletOutputStream outputStream;
+ private EscapedServletOutputStream jsonStream;
private int errorCode = SC_OK;
private String errorMessage;
+ /** Specify hash key for json concat **/
+ private String jsonVar = null;
+ private String url = null;
- protected ResponseWrapper(HttpServletResponse httpServletResponse) {
+ protected ResponseWrapper(HttpServletResponse httpServletResponse,
+ String jsonVar) throws IOException {
super(httpServletResponse);
+ if (jsonVar != null && jsonVar.length() > 0) {
+ this.jsonVar = jsonVar;
+ super.getOutputStream().println(jsonVar + "={");
+ }
}
+
@Override
public ServletOutputStream getOutputStream() throws IOException {
// For errors, we don't want the content returned by the remote
// server; we'll just include an HTTP error code to avoid creating
// syntactically invalid output overall.
if (errorCode != SC_OK) {
+ closeStream();
outputStream = new NullServletOutputStream();
}
-
+
if (outputStream == null) {
outputStream = super.getOutputStream();
}
+
return outputStream;
}
+ /**
+ * Restart a new file to concat
+ * Close previous file, and add start comment if not json concat
+ */
+ public void processUrl(String fileUrl) throws IOException {
+ closeStream();
+ errorCode = SC_OK;
+ this.url = fileUrl;
+ if (jsonVar == null) {
+ super.getOutputStream().println("/* ---- Start " + url + " ---- */");
+ } else {
+ // Create escaping stream (make sure url variable is defined)
+ jsonStream = new EscapedServletOutputStream();
+ outputStream = jsonStream;
+ }
+ }
+
+ /**
+ * Add close of json hash
+ */
+ public void done() throws IOException {
+ closeStream();
+ if (jsonVar != null) {
+ // Close json concat main variable
+ super.getOutputStream().println("};");
+ }
+ }
+
+ private void closeStream() throws IOException {
+ if (jsonVar == null && outputStream != null) {
+ outputStream.println("/* ---- End " + url + " ---- */");
+ } else if (jsonStream != null) {
+ byte[] data = jsonStream.getBytes();
+ ServletOutputStream mainStream = super.getOutputStream();
+ mainStream.print("\"" + url + "\":\"");
+ mainStream.write(data);
+ mainStream.println("\",");
+ }
+ outputStream = null;
+ jsonStream = null;
+ }
+
public int getStatus() {
return errorCode;
}
@@ -295,5 +369,42 @@
public void write(byte b[]) throws IOException {
}
}
+
+ /**
+ * Override Servlet output stream to support json escaping of the stream data
+ * Use getBytes to get the escaped data.
+ */
+ private static class EscapedServletOutputStream extends ServletOutputStream {
+
+ private ByteArrayOutputStream tempStream;
+ protected EscapedServletOutputStream() {
+ tempStream = new ByteArrayOutputStream();
+ }
+
+ public byte[] getBytes() throws IOException {
+ try {
+ return StringEscapeUtils.escapeJavaScript(tempStream.toString("UTF8")).getBytes();
+ } catch (UnsupportedEncodingException e) {
+ // Need to return IOException since that what ServletOutputStream constructor do.
+ throw new IOException("Unsuported encoding in data");
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ tempStream.write(b);
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ tempStream.write(b, off, len);
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ tempStream.write(b);
+ }
+ }
+
}
Added: incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/ConcatProxyServletTest.java
URL: http://svn.apache.org/viewvc/incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/ConcatProxyServletTest.java?rev=898140&view=auto
==============================================================================
--- incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/ConcatProxyServletTest.java (added)
+++ incubator/shindig/trunk/java/gadgets/src/test/java/org/apache/shindig/gadgets/servlet/ConcatProxyServletTest.java Tue Jan 12 00:51:08 2010
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.shindig.gadgets.servlet;
+
+import static org.easymock.EasyMock.expect;
+
+import org.apache.shindig.common.uri.Uri;
+import org.apache.shindig.gadgets.GadgetException;
+import org.apache.shindig.gadgets.http.HttpRequest;
+import org.apache.shindig.gadgets.http.HttpResponse;
+import org.apache.shindig.gadgets.http.HttpResponseBuilder;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConcatProxyServletTest extends ServletTestFixture {
+ private static final String REQUEST_DOMAIN = "example.org";
+
+ private static final Uri URL1 = Uri.parse("http://example.org/1.js");
+ private static final Uri URL2 = Uri.parse("http://example.org/2.js");
+ private static final Uri URL3 = Uri.parse("http://example.org/3.js");
+
+ private static final String SCRT1 = "var v1 = 1;";
+ private static final String SCRT2 = "var v2 = { \"a-b\": 1 , c: \"hello!,\" };";
+ private static final String SCRT3 = "var v3 = \"world\";";
+
+ private static final String SCRT1_ESCAPED = "var v1 = 1;";
+ private static final String SCRT2_ESCAPED =
+ "var v2 = { \\\"a-b\\\": 1 , c: \\\"hello!,\\\" };";
+ private static final String SCRT3_ESCAPED = "var v3 = \\\"world\\\";";
+
+ private final ProxyHandler proxyHandler =
+ new ProxyHandler(pipeline, lockedDomainService, null);
+ private final ConcatProxyServlet servlet = new ConcatProxyServlet();
+
+ @Before
+ public void setUp() throws Exception {
+ servlet.setProxyHandler(proxyHandler);
+ expect(request.getHeader("Host")).andReturn(REQUEST_DOMAIN).anyTimes();
+ expect(lockedDomainService.isSafeForOpenProxy(REQUEST_DOMAIN))
+ .andReturn(true).anyTimes();
+
+ expectGetAndReturnData(URL1,SCRT1);
+ expectGetAndReturnData(URL2,SCRT2);
+ expectGetAndReturnData(URL3,SCRT3);
+ }
+
+ private void expectGetAndReturnData(Uri url, String data) throws Exception {
+ HttpRequest req = new HttpRequest(url);
+ HttpResponse resp = new HttpResponseBuilder().setResponse(data.getBytes()).create();
+ expect(pipeline.execute(req)).andReturn(resp).anyTimes();
+ }
+
+ /**
+ * Simulate the added comments by concat
+ * @param data - concatenated data
+ * @param url - data source url
+ * @return data with added comments
+ */
+ private String addComment(String data, String url) {
+ String res = "/* ---- Start " + url + " ---- */\r\n"
+ + data + "/* ---- End " + url + " ---- */\r\n";
+ return res;
+ }
+
+ /**
+ * Simulate the asJSON result of one script
+ * @param url - the script url
+ * @param data - the script escaped content
+ * @return simulated hash mapping
+ */
+ private String addVar(String url, String data) {
+ return "\"" + url + "\":\"" + data +"\",\r\n";
+
+ }
+
+ /**
+ * Run a concat test
+ * @param result - expected concat results
+ * @param uris - list of uris to concat
+ * @throws Exception
+ */
+ private void runConcat(String result, Uri... uris) throws Exception {
+ for (int i = 0 ; i < uris.length ; i++) {
+ expect(request.getParameter(Integer.toString(i+1))).andReturn(uris[i].toString()).once();
+ }
+ expect(request.getParameter(Integer.toString(uris.length+1))).andReturn(null).once();
+ replay();
+ // Run the servlet
+ servlet.doGet(request, recorder);
+ verify();
+ assertEquals(result, recorder.getResponseAsString());
+ assertEquals(200, recorder.getHttpStatusCode());
+ }
+
+ @Test
+ public void testSimpleConcat() throws Exception {
+ String results = addComment(SCRT1, URL1.toString()) + addComment(SCRT2,URL2.toString());
+ runConcat(results, URL1,URL2);
+ }
+
+ @Test
+ public void testThreeConcat() throws Exception {
+ String results = addComment(SCRT1, URL1.toString()) + addComment(SCRT2,URL2.toString())
+ + addComment(SCRT3,URL3.toString());
+ runConcat(results, URL1, URL2, URL3);
+ }
+
+ @Test
+ public void testConcatBadException() throws Exception {
+ final Uri URL4 = Uri.parse("http://example.org/4.js");
+
+ HttpRequest req = new HttpRequest(URL4);
+ expect(pipeline.execute(req)).andThrow(
+ new GadgetException(GadgetException.Code.HTML_PARSE_ERROR)).anyTimes();
+
+ String results = addComment(SCRT1, URL1.toString())
+ + "/* ---- Start http://example.org/4.js ---- */\r\n"
+ + "HTML_PARSE_ERROR concat(http://example.org/4.js) null";
+
+ expect(request.getParameter(Integer.toString(1))).andReturn(URL1.toString()).once();
+ expect(request.getParameter(Integer.toString(2))).andReturn(URL4.toString()).once();
+ replay();
+ // Run the servlet
+ servlet.doGet(request, recorder);
+ verify();
+ assertEquals(results, recorder.getResponseAsString());
+ assertEquals(400, recorder.getHttpStatusCode());
+ }
+
+ @Test
+ public void testAsJsonConcat() throws Exception {
+ expect(request.getParameter("json")).andReturn("_js").once();
+ String results = "_js={\r\n"
+ + addVar(URL1.toString(), SCRT1_ESCAPED)
+ + addVar(URL2.toString(), SCRT2_ESCAPED)
+ + "};\r\n";
+ runConcat(results, URL1, URL2);
+ }
+
+ @Test
+ public void testThreeAsJsonConcat() throws Exception {
+ expect(request.getParameter("json")).andReturn("testJs").once();
+ String results = "testJs={\r\n"
+ + addVar(URL1.toString(), SCRT1_ESCAPED)
+ + addVar(URL2.toString(), SCRT2_ESCAPED)
+ + addVar(URL3.toString(), SCRT3_ESCAPED)
+ + "};\r\n";
+ runConcat(results, URL1, URL2, URL3);
+ }
+
+ @Test
+ public void testBadJsonVarConcat() throws Exception {
+ expect(request.getParameter("json")).andReturn("bad code;").once();
+ replay();
+ servlet.doGet(request, recorder);
+ verify();
+ String results = "/* ---- Error 400, Bad json variable name bad code; ---- */\r\n";
+ assertEquals(results, recorder.getResponseAsString());
+ assertEquals(400, recorder.getHttpStatusCode());
+ }
+
+ @Test
+ public void testAsJsonConcat404() throws Exception {
+ final Uri URL4 = Uri.parse("http://example.org/4.js");
+
+ HttpRequest req = new HttpRequest(URL4);
+ HttpResponse resp = new HttpResponseBuilder().setHttpStatusCode(404).create();
+ expect(pipeline.execute(req)).andReturn(resp).anyTimes();
+
+ expect(request.getParameter("json")).andReturn("_js").once();
+ String results = "_js={\r\n"
+ + addVar(URL1.toString(), SCRT1_ESCAPED)
+ + addVar(URL4.toString(),"")
+ + "/* ---- Error 404 ---- */\r\n"
+ + "};\r\n";
+ runConcat(results, URL1, URL4);
+ }
+
+ @Test
+ public void testAsJsonConcatException() throws Exception {
+ final Uri URL4 = Uri.parse("http://example.org/4.js");
+
+ HttpRequest req = new HttpRequest(URL4);
+ expect(pipeline.execute(req)).andThrow(
+ new GadgetException(GadgetException.Code.FAILED_TO_RETRIEVE_CONTENT)).anyTimes();
+
+ expect(request.getParameter("json")).andReturn("_js").once();
+ String results = "_js={\r\n"
+ + addVar(URL1.toString(), SCRT1_ESCAPED)
+ + "/* ---- End http://example.org/4.js 404 ---- */\r\n"
+ + addVar(URL4.toString(),"")
+ + "};\r\n";
+ runConcat(results, URL1, URL4);
+ }
+
+ @Test
+ public void testAsJsonConcatBadException() throws Exception {
+ final Uri URL4 = Uri.parse("http://example.org/4.js");
+
+ HttpRequest req = new HttpRequest(URL4);
+ expect(pipeline.execute(req)).andThrow(
+ new GadgetException(GadgetException.Code.HTML_PARSE_ERROR)).anyTimes();
+
+ expect(request.getParameter("json")).andReturn("_js").once();
+ String results = "_js={\r\n"
+ + addVar(URL1.toString(), SCRT1_ESCAPED)
+ + addVar(URL4.toString(),"")
+ + "};\r\n"
+ + "HTML_PARSE_ERROR concat(http://example.org/4.js) null";
+
+ expect(request.getParameter(Integer.toString(1))).andReturn(URL1.toString()).once();
+ expect(request.getParameter(Integer.toString(2))).andReturn(URL4.toString()).once();
+ replay();
+ // Run the servlet
+ servlet.doGet(request, recorder);
+ verify();
+ assertEquals(results, recorder.getResponseAsString());
+ assertEquals(400, recorder.getHttpStatusCode());
+ }
+
+}