You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/01/03 04:35:19 UTC
svn commit: r1555005 - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/
hadoop-yarn/ha...
Author: vinodkv
Date: Fri Jan 3 03:35:18 2014
New Revision: 1555005
URL: http://svn.apache.org/r1555005
Log:
YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory server. Contributed by Mayank Bansal.
Added:
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java
Modified:
hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt Fri Jan 3 03:35:18 2014
@@ -460,6 +460,9 @@ Branch YARN-321: Generic ApplicationHist
YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv)
+ YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory
+ server. (Mayank Bansal via vinodkv)
+
Release 2.1.1-beta - UNRELEASED
YARN-948. Changed ResourceManager to validate the release container list
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java Fri Jan 3 03:35:18 2014
@@ -178,4 +178,8 @@ public final class StringHelper {
public static String percent(double value) {
return String.format("%.2f", value * 100);
}
+
+ public static String getPartUrl(String url, String part) {
+ return url.substring(url.indexOf(part));
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java Fri Jan 3 03:35:18 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.webapp.util;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -27,7 +29,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.base.Joiner;
@@ -170,4 +174,11 @@ public class WebAppUtils {
return schemePrefix + url;
}
}
+
+ public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
+ ContainerId containerId, String user) {
+ return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/",
+ allocatedNode, "/", ConverterUtils.toString(containerId), "/",
+ ConverterUtils.toString(containerId), "/", user);
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java Fri Jan 3 03:35:18 2014
@@ -47,4 +47,10 @@ public class AHSController extends Contr
render(ContainerPage.class);
}
+ /**
+ * Render the logs page.
+ */
+ public void logs() {
+ render(AHSLogsPage.class);
+ }
}
Added: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java?rev=1555005&view=auto
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java (added)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSLogsPage.java Fri Jan 3 03:35:18 2014
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.ENTITY_STRING;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.log.AggregatedLogsBlock;
+
+public class AHSLogsPage extends AHSView {
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSView#
+ * preHead(org.apache.hadoop .yarn.webapp.hamlet.Hamlet.HTML)
+ */
+ @Override
+ protected void preHead(Page.HTML<_> html) {
+ String logEntity = $(ENTITY_STRING);
+ if (logEntity == null || logEntity.isEmpty()) {
+ logEntity = $(CONTAINER_ID);
+ }
+ if (logEntity == null || logEntity.isEmpty()) {
+ logEntity = "UNKNOWN";
+ }
+ commonPreHead(html);
+ }
+
+ /**
+ * The content of this page is the AggregatedLogsBlock
+ *
+ * @return AggregatedLogsBlock.class
+ */
+ @Override
+ protected Class<? extends SubView> content() {
+ return AggregatedLogsBlock.class;
+ }
+}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java Fri Jan 3 03:35:18 2014
@@ -45,5 +45,7 @@ public class AHSWebApp extends WebApp im
route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), AHSController.class,
"appattempt");
route(pajoin("/container", CONTAINER_ID), AHSController.class, "container");
+ route(pajoin("/logs", NM_NODENAME, CONTAINER_ID, ENTITY_STRING, APP_OWNER,
+ CONTAINER_LOG_TYPE), AHSController.class, "logs");
}
}
\ No newline at end of file
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java Fri Jan 3 03:35:18 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID;
import java.io.IOException;
@@ -126,6 +127,8 @@ public class AppAttemptBlock extends Htm
StringBuilder containersTableData = new StringBuilder("[\n");
for (ContainerReport containerReport : containers) {
+ String logURL = containerReport.getLogUrl();
+ logURL = getPartUrl(logURL, "log");
ContainerInfo container = new ContainerInfo(containerReport);
// ConatinerID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
containersTableData
@@ -141,9 +144,9 @@ public class AppAttemptBlock extends Htm
.append("</a>\",\"")
.append(container.getContainerExitStatus())
.append("\",\"<a href='")
- .append(container.getLogUrl() == null ? "#" : url(container.getLogUrl()))
+ .append(logURL == null ? "#" : url(logURL))
.append("'>")
- .append(container.getLogUrl() == null ? "N/A" : "Logs")
+ .append(logURL == null ? "N/A" : "Logs")
.append("</a>\"],\n");
}
if (containersTableData.charAt(containersTableData.length() - 2) == ',') {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java Fri Jan 3 03:35:18 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
import java.io.IOException;
@@ -147,7 +148,8 @@ public class AppBlock extends HtmlBlock
if (containerReport != null) {
ContainerInfo container = new ContainerInfo(containerReport);
startTime = container.getStartedTime();
- logsLink = container.getLogUrl();
+ logsLink = containerReport.getLogUrl();
+ logsLink = getPartUrl(logsLink,"log");
}
String nodeLink = null;
if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java Fri Jan 3 03:35:18 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
import java.io.IOException;
@@ -76,8 +77,10 @@ public class ContainerBlock extends Html
puts("Container not found: " + containerid);
return;
}
+
ContainerInfo container = new ContainerInfo(containerReport);
-
+ String logURL = containerReport.getLogUrl();
+ logURL = getPartUrl(logURL,"log");
setTitle(join("Container ", containerid));
info("Container Overview").
@@ -91,8 +94,7 @@ public class ContainerBlock extends Html
container.getFinishedTime()))).
_("Resource:", container.getAllocatedMB() + " Memory, " +
container.getAllocatedVCores() + " VCores").
- _("Logs:", container.getLogUrl() == null ? "#" : root_url(container.getLogUrl()),
- container.getLogUrl() == null ? "N/A" : container.getLogUrl()).
+ _("Logs:", logURL == null ? "#" : url(logURL), "Logs").
_("Diagnostics:", container.getDiagnosticsInfo());
html._(InfoBlock.class);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1555005&r1=1555004&r2=1555005&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Fri Jan 3 03:35:18 2014
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
-import static org.apache.hadoop.yarn.util.StringHelper.join;
-
import java.util.EnumSet;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -27,7 +25,6 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -46,7 +43,7 @@ import org.apache.hadoop.yarn.state.Inva
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@SuppressWarnings({"unchecked", "rawtypes"})
public class RMContainerImpl implements RMContainer {
@@ -366,10 +363,9 @@ public class RMContainerImpl implements
public void transition(RMContainerImpl container, RMContainerEvent event) {
// The logs of running containers should be found on NM webUI
// The logs should be accessible after the container is launched
- container.logURL = join(HttpConfig.getSchemePrefix(),
- container.container.getNodeHttpAddress(), "/node", "/containerlogs/",
- ConverterUtils.toString(container.containerId), "/",
- container.user);
+ container.logURL = WebAppUtils.getLogUrl(container.container
+ .getNodeHttpAddress(), container.getAllocatedNode().toString(),
+ container.containerId, container.user);
// Unregister from containerAllocationExpirer.
container.containerAllocationExpirer.unregister(container
.getContainerId());