You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tg...@apache.org on 2012/03/16 21:10:51 UTC
svn commit: r1301731 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/
Author: tgraves
Date: Fri Mar 16 20:10:51 2012
New Revision: 1301731
URL: http://svn.apache.org/viewvc?rev=1301731&view=rev
Log:
MAPREDUCE-4006. History server container log web UI sometimes combines stderr/stdout/syslog contents together (Siddharth Seth via tgraves)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1301731&r1=1301730&r2=1301731&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Mar 16 20:10:51 2012
@@ -328,6 +328,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-4005. Fixes broken AM container logs URL on ResourceManager
Application Info page. (Jason Lowe via sseth)
+ MAPREDUCE-4006. History server container log web UI sometimes combines
+ stderr/stdout/syslog contents together (Siddharth Seth via tgraves)
+
Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java?rev=1301731&r1=1301730&r2=1301731&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java Fri Mar 16 20:10:51 2012
@@ -430,7 +430,7 @@ public class AggregatedLogFormat {
int currentRead = 0;
int totalRead = 0;
while ((currentRead = reader.read(cbuf, 0, bufferSize)) != -1) {
- writer.write(cbuf);
+ writer.write(cbuf, 0, currentRead);
totalRead += currentRead;
}
}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java?rev=1301731&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java Fri Mar 16 20:10:51 2012
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Collections;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAggregatedLogFormat {
+
+ private static final File testWorkDir = new File("target",
+ "TestAggregatedLogFormat");
+ private static final Configuration conf = new Configuration();
+ private static final FileSystem fs;
+ private static final char filler = 'x';
+ private static final Log LOG = LogFactory
+ .getLog(TestAggregatedLogFormat.class);
+
+ static {
+ try {
+ fs = FileSystem.get(conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Before
+ @After
+ public void cleanupTestDir() throws Exception {
+ Path workDirPath = new Path(testWorkDir.getAbsolutePath());
+ LOG.info("Cleaning test directory [" + workDirPath + "]");
+ fs.delete(workDirPath, true);
+ }
+
+ //Verify the output generated by readAContainerLogs(DataInputStream, Writer)
+ @Test
+ public void testReadAcontainerLogs1() throws Exception {
+ Configuration conf = new Configuration();
+ File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
+ Path remoteAppLogFile =
+ new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
+ Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
+ ContainerId testContainerId = BuilderUtils.newContainerId(1, 1, 1, 1);
+ Path t =
+ new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
+ .getApplicationId().toString());
+ Path srcFilePath = new Path(t, testContainerId.toString());
+
+ int numChars = 80000;
+
+ writeSrcFile(srcFilePath, "stdout", numChars);
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);
+
+ LogKey logKey = new LogKey(testContainerId);
+ LogValue logValue =
+ new LogValue(Collections.singletonList(srcFileRoot.toString()),
+ testContainerId);
+
+ logWriter.append(logKey, logValue);
+ logWriter.closeWriter();
+
+ LogReader logReader = new LogReader(conf, remoteAppLogFile);
+ LogKey rLogKey = new LogKey();
+ DataInputStream dis = logReader.next(rLogKey);
+ Writer writer = new StringWriter();
+ LogReader.readAcontainerLogs(dis, writer);
+
+ String s = writer.toString();
+ int expectedLength =
+ "\n\nLogType:stdout".length() + ("\nLogLength:" + numChars).length()
+ + "\nLog Contents:\n".length() + numChars;
+ Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
+ Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
+ Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0 ; i < numChars ; i++) {
+ sb.append(filler);
+ }
+ String expectedContent = sb.toString();
+ Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
+
+ Assert.assertEquals(expectedLength, s.length());
+ }
+
+ private void writeSrcFile(Path srcFilePath, String fileName, long length)
+ throws IOException {
+ File dir = new File(srcFilePath.toString());
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new IOException("Unable to create directory : " + dir);
+ }
+ }
+ File outputFile = new File(new File(srcFilePath.toString()), fileName);
+ FileOutputStream os = new FileOutputStream(outputFile);
+ OutputStreamWriter osw = new OutputStreamWriter(os, "UTF8");
+ int ch = filler;
+ for (int i = 0; i < length; i++) {
+ osw.write(ch);
+ }
+ osw.close();
+ }
+}