You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by je...@apache.org on 2013/06/20 20:56:13 UTC
svn commit: r1495129 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/
hadoop-yarn/hadoo...
Author: jeagles
Date: Thu Jun 20 18:56:12 2013
New Revision: 1495129
URL: http://svn.apache.org/r1495129
Log:
YARN-478. fix coverage org.apache.hadoop.yarn.webapp.log (Aleksey Gorshkov via jeagles)
Added:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/BlockForTest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/HtmlBlockForTest.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1495129&r1=1495128&r2=1495129&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Jun 20 18:56:12 2013
@@ -989,6 +989,9 @@ Release 0.23.9 - UNRELEASED
YARN-427. Coverage fix for org.apache.hadoop.yarn.server.api.* (Aleksey
Gorshkov via jeagles)
+
+ YARN-478. fix coverage org.apache.hadoop.yarn.webapp.log (Aleksey Gorshkov
+ via jeagles)
OPTIMIZATIONS
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java?rev=1495129&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java Thu Jun 20 18:56:12 2013
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.log.AggregatedLogsBlockForTest;
+import org.apache.hadoop.yarn.webapp.view.BlockForTest;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlockForTest;
+import org.junit.Test;
+
+import static org.mockito.Mockito.*;
+import static org.junit.Assert.*;
+
+/**
+ * Test AggregatedLogsBlock. AggregatedLogsBlock should check user, aggregate a
+ * logs into one file and show this logs or errors into html code
+ *
+ */
+public class TestAggregatedLogsBlock {
+ /**
+ * Bad user. User 'owner' is trying to read logs without access
+ */
+ @Test
+ public void testAccessDenied() throws Exception {
+
+ FileUtil.fullyDelete(new File("target/logs"));
+ Configuration configuration = getConfiguration();
+
+ writeLogs("target/logs/logs/application_0_0001/container_0_0001_01_000001");
+
+ writeLog(configuration, "owner");
+
+ AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest(
+ configuration, "owner", "container_0_0001_01_000001");
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(data);
+ HtmlBlock html = new HtmlBlockForTest();
+ HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
+ aggregatedBlock.render(block);
+
+ block.getWriter().flush();
+ String out = data.toString();
+ assertTrue(out
+ .contains("User [owner] is not authorized to view the logs for entity"));
+
+ }
+
+ /**
+ * try to read bad logs
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testBadLogs() throws Exception {
+
+ FileUtil.fullyDelete(new File("target/logs"));
+ Configuration configuration = getConfiguration();
+
+ writeLogs("target/logs/logs/application_0_0001/container_0_0001_01_000001");
+
+ writeLog(configuration, "owner");
+
+ AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest(
+ configuration, "admin", "container_0_0001_01_000001");
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(data);
+ HtmlBlock html = new HtmlBlockForTest();
+ HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
+ aggregatedBlock.render(block);
+
+ block.getWriter().flush();
+ String out = data.toString();
+ assertTrue(out
+ .contains("Logs not available for entity. Aggregation may not be complete, Check back later or try the nodemanager at localhost:1234"));
+
+ }
+
+ /**
+ * All ok and the AggregatedLogsBlockFor should aggregate logs and show it.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testAggregatedLogsBlock() throws Exception {
+
+ FileUtil.fullyDelete(new File("target/logs"));
+ Configuration configuration = getConfiguration();
+
+ writeLogs("target/logs/logs/application_0_0001/container_0_0001_01_000001");
+
+ writeLog(configuration, "admin");
+
+ AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest(
+ configuration, "admin", "container_0_0001_01_000001");
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(data);
+ HtmlBlock html = new HtmlBlockForTest();
+ HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
+ aggregatedBlock.render(block);
+
+ block.getWriter().flush();
+ String out = data.toString();
+ assertTrue(out.contains("test log1"));
+ assertTrue(out.contains("test log2"));
+ assertTrue(out.contains("test log3"));
+
+ }
+ /**
+ * Log files was deleted.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testNoLogs() throws Exception {
+
+ FileUtil.fullyDelete(new File("target/logs"));
+ Configuration configuration = getConfiguration();
+
+ File f = new File("target/logs/logs/application_0_0001/container_0_0001_01_000001");
+ if (!f.exists()) {
+ assertTrue(f.mkdirs());
+ }
+ writeLog(configuration, "admin");
+
+ AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest(
+ configuration, "admin", "container_0_0001_01_000001");
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(data);
+ HtmlBlock html = new HtmlBlockForTest();
+ HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
+ aggregatedBlock.render(block);
+
+ block.getWriter().flush();
+ String out = data.toString();
+ assertTrue(out.contains("No logs available for container container_0_0001_01_000001"));
+
+ }
+
+
+ private Configuration getConfiguration() {
+ Configuration configuration = new Configuration();
+ configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+ configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, "target/logs");
+ configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+ return configuration;
+ }
+
+ private AggregatedLogsBlockForTest getAggregatedLogsBlockForTest(
+ Configuration configuration, String user, String containerId) {
+ HttpServletRequest request = mock(HttpServletRequest.class);
+ when(request.getRemoteUser()).thenReturn(user);
+ AggregatedLogsBlockForTest aggregatedBlock = new AggregatedLogsBlockForTest(
+ configuration);
+ aggregatedBlock.setRequest(request);
+ aggregatedBlock.moreParams().put(YarnWebParams.CONTAINER_ID, containerId);
+ aggregatedBlock.moreParams().put(YarnWebParams.NM_NODENAME,
+ "localhost:1234");
+ aggregatedBlock.moreParams().put(YarnWebParams.APP_OWNER, user);
+ aggregatedBlock.moreParams().put("start", "");
+ aggregatedBlock.moreParams().put("end", "");
+ aggregatedBlock.moreParams().put(YarnWebParams.ENTITY_STRING, "entity");
+ return aggregatedBlock;
+ }
+
+ private void writeLog(Configuration configuration, String user)
+ throws Exception {
+ ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1);
+ ContainerId containerId = ContainerIdPBImpl.newInstance(appAttemptId, 1);
+
+ String path = "target/logs/" + user
+ + "/logs/application_0_0001/localhost_1234";
+ File f = new File(path);
+ if (!f.getParentFile().exists()) {
+ assertTrue(f.getParentFile().mkdirs());
+ }
+ List<String> rootLogDirs = Arrays.asList("target/logs/logs");
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
+ AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(
+ configuration, new Path(path), ugi);
+ writer.writeApplicationOwner(ugi.getUserName());
+
+ Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
+ appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName());
+ writer.writeApplicationACLs(appAcls);
+
+ writer.append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"),
+ new AggregatedLogFormat.LogValue(rootLogDirs, containerId,UserGroupInformation.getCurrentUser().getShortUserName()));
+ writer.closeWriter();
+ }
+
+ private void writeLogs(String dirName) throws Exception {
+ File f = new File(dirName + File.separator + "log1");
+ if (!f.getParentFile().exists()) {
+ assertTrue(f.getParentFile().mkdirs());
+ }
+
+ writeLog(dirName + File.separator + "log1", "test log1");
+ writeLog(dirName + File.separator + "log2", "test log2");
+ writeLog(dirName + File.separator + "log3", "test log3");
+ }
+
+ private void writeLog(String fileName, String text) throws Exception {
+ File f = new File(fileName);
+ Writer writer = new FileWriter(f);
+ writer.write(text);
+ writer.flush();
+ writer.close();
+ }
+
+}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java?rev=1495129&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java Thu Jun 20 18:56:12 2013
@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.webapp.log;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class AggregatedLogsBlockForTest extends AggregatedLogsBlock {
+
+ final private Map<String, String> params = new HashMap<String, String>();
+ private HttpServletRequest request;
+ public AggregatedLogsBlockForTest(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public void render(Block html) {
+ super.render(html);
+ }
+
+ public Map<String, String> moreParams() {
+ return params;
+ }
+
+ public HttpServletRequest request() {
+ return request;
+ }
+ public void setRequest(HttpServletRequest request) {
+ this.request = request;
+ }
+
+}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/BlockForTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/BlockForTest.java?rev=1495129&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/BlockForTest.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/BlockForTest.java Thu Jun 20 18:56:12 2013
@@ -0,0 +1,30 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.webapp.view;
+
+import java.io.PrintWriter;
+
+public class BlockForTest extends HtmlBlock.Block {
+
+ public BlockForTest(HtmlBlock htmlBlock, PrintWriter out, int level,
+ boolean wasInline) {
+ htmlBlock.super(out, level, wasInline);
+ }
+
+}
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/HtmlBlockForTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/HtmlBlockForTest.java?rev=1495129&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/HtmlBlockForTest.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/HtmlBlockForTest.java Thu Jun 20 18:56:12 2013
@@ -0,0 +1,27 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.webapp.view;
+
+public class HtmlBlockForTest extends HtmlBlock {
+
+ @Override
+ protected void render(Block html) {
+ info("test!");
+ }
+}