You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2011/01/26 20:34:24 UTC
svn commit: r1063846 - in /hadoop/mapreduce/trunk: CHANGES.txt ivy.xml
src/java/org/apache/hadoop/mapred/TaskLogServlet.java
src/java/org/apache/hadoop/mapred/TaskTracker.java
src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java
Author: todd
Date: Wed Jan 26 19:34:24 2011
New Revision: 1063846
URL: http://svn.apache.org/viewvc?rev=1063846&view=rev
Log:
MAPREDUCE-2253. Servlets should specify content type. Contributed by Todd Lipcon
Added:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/ivy.xml
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1063846&r1=1063845&r2=1063846&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Jan 26 19:34:24 2011
@@ -498,6 +498,8 @@ Release 0.22.0 - Unreleased
MAPREDUCE-2277. TestCapacitySchedulerWithJobTracker needs to wait for jobs to
complete before testing status. (todd)
+ MAPREDUCE-2253. Servlets should specify content type (todd)
+
Release 0.21.1 - Unreleased
NEW FEATURES
Modified: hadoop/mapreduce/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/ivy.xml?rev=1063846&r1=1063845&r2=1063846&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/ivy.xml (original)
+++ hadoop/mapreduce/trunk/ivy.xml Wed Jan 26 19:34:24 2011
@@ -105,6 +105,8 @@
</dependency>
<dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}"
conf="test->default"/>
+ <dependency org="org.mortbay.jetty" name="jetty-servlet-tester" rev="${jetty.version}"
+ conf="test->default"/>
<!-- dependency addition for the fault injection -->
<dependency org="org.aspectj" name="aspectjrt" rev="${aspectj.version}"
conf="common->default"/>
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogServlet.java?rev=1063846&r1=1063845&r2=1063846&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogServlet.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskLogServlet.java Wed Jan 26 19:34:24 2011
@@ -264,6 +264,7 @@ public class TaskLogServlet extends Http
OutputStream out = response.getOutputStream();
if( !plainText ) {
+ response.setContentType("text/html; charset=utf-8");
out.write(("<html>\n" +
"<title>Task Logs: '" + attemptId + "'</title>\n" +
"<body>\n" +
@@ -295,6 +296,7 @@ public class TaskLogServlet extends Http
response.sendError(HttpServletResponse.SC_BAD_REQUEST,
"You must supply a value for `filter' (STDOUT, STDERR, or SYSLOG) if you set plainText = true");
} else {
+ response.setContentType("text/plain; charset=utf-8");
printTaskLog(response, out, attemptId, start, end, plainText, filter,
isCleanup);
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1063846&r1=1063845&r2=1063846&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed Jan 26 19:34:24 2011
@@ -3667,6 +3667,8 @@ public class TaskTracker
int numMaps = 0;
try {
shuffleMetrics.serverHandlerBusy();
+ response.setContentType("application/octet-stream");
+
outStream = new DataOutputStream(response.getOutputStream());
//use the same buffersize as used for reading the data from disk
response.setBufferSize(MAX_BYTES_TO_READ);
Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java?rev=1063846&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java (added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskLogServlet.java Wed Jan 26 19:34:24 2011
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.jetty.testing.HttpTester;
+import org.mortbay.jetty.testing.ServletTester;
+
+public class TestTaskLogServlet {
+ private static final Log LOG = LogFactory.getLog(TestTaskLogServlet.class);
+ private ServletTester tester;
+
+ @Before
+ public void setup() throws Exception {
+ tester = new ServletTester();
+ tester.setContextPath("/");
+ tester.addServlet(TaskLogServlet.class, "/tasklog");
+ tester.start();
+ }
+
+ @Test
+ public void testMissingParameters() throws Exception {
+ HttpTester request = new HttpTester();
+ request.setMethod("GET");
+ request.setURI("/tasklog");
+ request.setVersion("HTTP/1.0");
+
+ HttpTester response = new HttpTester();
+ response.parse(tester.getResponses(request.generate()));
+
+ assertEquals(400,response.getStatus());
+ }
+
+ private void setupValidLogs(String attemptIdStr) throws IOException {
+ TaskAttemptID attemptId = TaskAttemptID.forName(attemptIdStr);
+ File logDir = TaskLog.getAttemptDir(attemptId, false);
+ FileUtil.fullyDelete(logDir);
+ logDir.mkdirs();
+ assertTrue(logDir.exists());
+
+ // Now make the logs with some HTML in the output
+ TaskLog.syncLogs(logDir.getAbsolutePath(), attemptId, false);
+ makeLog(new File(logDir, "stderr"), "<b>this is stderr");
+ makeLog(new File(logDir, "stdout"), "<b>this is stdout");
+ makeLog(new File(logDir, "syslog"), "<b>this is syslog");
+ TaskLog.syncLogs(logDir.getAbsolutePath(), attemptId, false);
+ }
+
+ @Test
+ public void testHtmlLogs() throws Exception {
+ String attemptIdStr = "attempt_123_0001_m_000001_0";
+ setupValidLogs(attemptIdStr);
+
+ HttpTester request = new HttpTester();
+ request.setMethod("GET");
+ request.setURI("/tasklog?attemptid=" + attemptIdStr);
+ request.setVersion("HTTP/1.0");
+
+ // Make sure all the contents show up and properly escaped
+ HttpTester response = doRequest(request);
+ assertEquals(HttpServletResponse.SC_OK, response.getStatus());
+ assertEquals("text/html; charset=utf-8", response.getHeader("content-type"));
+ assertTrue(response.getContent().contains("<b>this is stderr"));
+ assertTrue(response.getContent().contains("<b>this is stdout"));
+ assertTrue(response.getContent().contains("<b>this is syslog"));
+
+ // Only read a small chunk of each file <***b>thi***s
+ // (should still be escaped)
+ request.setURI("/tasklog?attemptid=" + attemptIdStr
+ + "&start=1&end=6");
+ response = doRequest(request);
+ assertEquals(HttpServletResponse.SC_OK, response.getStatus());
+ assertEquals("text/html; charset=utf-8", response.getHeader("content-type"));
+ assertFalse(response.getContent().contains("<b"));
+ assertFalse(response.getContent().contains("this is"));
+ assertTrue(response.getContent().contains("b>thi</pre>"));
+ }
+
+ @Test
+ public void testPlaintextLogs() throws Exception {
+ String attemptIdStr = "attempt_123_0001_m_000001_0";
+ setupValidLogs(attemptIdStr);
+
+ HttpTester request = new HttpTester();
+ request.setMethod("GET");
+ request.setURI("/tasklog?plaintext=true&attemptid=" + attemptIdStr);
+ request.setVersion("HTTP/1.0");
+
+ // Make sure all the contents show up and properly escaped
+ HttpTester response = doRequest(request);
+ // Bad request because we require a 'filter'
+ assertEquals(HttpServletResponse.SC_BAD_REQUEST, response.getStatus());
+
+ // Try again with filter
+ request.setURI("/tasklog?plaintext=true&filter=stdout&attemptid=" + attemptIdStr);
+ response = doRequest(request);
+
+ // Response should be text/plain, not be escaped
+ assertEquals("text/plain; charset=utf-8", response.getHeader("content-type"));
+ assertEquals("<b>this is stdout", response.getContent());
+
+ // Test range request
+ request.setURI("/tasklog?plaintext=true&filter=stdout" +
+ "&attemptid=" + attemptIdStr +
+ "&start=1&end=6");
+ response = doRequest(request);
+
+ // Response should be text/plain, not be escaped
+ assertEquals("text/plain; charset=utf-8", response.getHeader("content-type"));
+ assertEquals("b>thi", response.getContent());
+ }
+
+ private HttpTester doRequest(HttpTester request) throws Exception {
+ String reqStr = request.generate();
+ LOG.info("Testing request: " + reqStr);
+ String respStr = tester.getResponses(reqStr);
+ LOG.info("Response: " + respStr);
+ HttpTester response = new HttpTester();
+ response.parse(respStr);
+ return response;
+ }
+
+ private void makeLog(File f, String contents) throws IOException {
+ LOG.info("Creating log at " + f);
+ FileWriter fw = new FileWriter(f);
+ try {
+ fw.write(contents);
+ } finally {
+ fw.close();
+ }
+ }
+
+}