You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/10/26 08:29:23 UTC
svn commit: r1189036 [1/2] - in
/hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapredu...
Author: acmurthy
Date: Wed Oct 26 06:29:21 2011
New Revision: 1189036
URL: http://svn.apache.org/viewvc?rev=1189036&view=rev
Log:
MAPREDUCE-2858. Added a WebApp Proxy for applications. Contributed by Robert Evans.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpPrincipal.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpServletRequestWrapper.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/pom.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/pom.xml
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Oct 26 06:29:21 2011
@@ -118,6 +118,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2708. Designed and implemented MR Application Master recovery to
make MR AMs resume their progress after restart. (Sharad Agarwal via vinodkv)
+ MAPREDUCE-2858. Added a WebApp Proxy for applications. (Robert Evans via
+ acmurthy)
+
IMPROVEMENTS
MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Wed Oct 26 06:29:21 2011
@@ -43,6 +43,10 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<scope>test</scope>
</dependency>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java Wed Oct 26 06:29:21 2011
@@ -341,6 +341,8 @@ class JobSubmitter {
Path submitJobDir = new Path(jobStagingArea, jobId.toString());
JobStatus status = null;
try {
+ conf.set("hadoop.http.filter.initializers",
+ "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, submitJobDir.toString());
LOG.debug("Configuring job " + jobId + " with " + submitJobDir
+ " as the submit dir");
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java Wed Oct 26 06:29:21 2011
@@ -77,7 +77,8 @@ public class NotRunningJob implements MR
// Setting AppState to NEW and finalStatus to UNDEFINED as they are never used
// for a non running job
return BuilderUtils.newApplicationReport(unknownAppId, "N/A", "N/A", "N/A", "N/A", 0, "",
- YarnApplicationState.NEW, "N/A", "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null);
+ YarnApplicationState.NEW, "N/A", "N/A", 0, 0,
+ FinalApplicationStatus.UNDEFINED, null, "N/A");
}
NotRunningJob(ApplicationReport applicationReport, JobState jobState) {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Wed Oct 26 06:29:21 2011
@@ -231,14 +231,14 @@ public class TestClientServiceDelegate {
return BuilderUtils.newApplicationReport(BuilderUtils.newApplicationId(
1234, 5), "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A");
}
private ApplicationReport getRunningApplicationReport(String host, int port) {
return BuilderUtils.newApplicationReport(BuilderUtils.newApplicationId(
1234, 5), "user", "queue", "appname", host, port, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.UNDEFINED, null);
+ FinalApplicationStatus.UNDEFINED, null, "N/A");
}
private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml Wed Oct 26 06:29:21 2011
@@ -54,6 +54,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<version>${project.version}</version>
</dependency>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh Wed Oct 26 06:29:21 2011
@@ -30,3 +30,5 @@ bin=`cd "$bin"; pwd`
"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR start nodemanager
# start historyserver
#"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start historyserver
+# start proxyserver
+#"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR start proxyserver
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh Wed Oct 26 06:29:21 2011
@@ -30,4 +30,5 @@ bin=`cd "$bin"; pwd`
"$bin"/yarn-daemons.sh --config $YARN_CONF_DIR stop nodemanager
# stop historyServer
"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop historyserver
-
+# stop proxy server
+"$bin"/yarn-daemon.sh --config $YARN_CONF_DIR stop proxyserver
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/yarn?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/bin/yarn Wed Oct 26 06:29:21 2011
@@ -206,6 +206,9 @@ elif [ "$COMMAND" = "nodemanager" ] ; th
elif [ "$COMMAND" = "historyserver" ] ; then
CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
YARN_OPTS="$YARN_OPTS $YARN_JOB_HISTORYSERVER_OPTS"
+elif [ "$COMMAND" = "proxyserver" ] ; then
+ CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
+ YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS"
elif [ "$COMMAND" = "version" ] ; then
CLASS=org.apache.hadoop.util.VersionInfo
YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Wed Oct 26 06:29:21 2011
@@ -60,6 +60,14 @@ public interface ApplicationConstants {
public static final String LOCAL_DIR_ENV = "YARN_LOCAL_DIRS";
+ /**
+ * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
+ * ApplicationMaster's environment only. This states that for all non-relative
+ * web URLs in the app masters web UI what base should they have.
+ */
+ public static final String APPLICATION_WEB_PROXY_BASE_ENV =
+ "APPLICATION_WEB_PROXY_BASE";
+
public static final String LOG_DIR_EXPANSION_VAR = "<LOG_DIR>";
public static final String STDERR = "stderr";
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Wed Oct 26 06:29:21 2011
@@ -174,6 +174,19 @@ public interface ApplicationReport {
@Private
@Unstable
void setTrackingUrl(String url);
+
+ /**
+ * Get the original not-proxied <em>tracking url</em> for the application.
+ * This is intended to only be used by the proxy itself.
+ * @return the original not-proxied <em>tracking url</em> for the application
+ */
+ @Private
+ @Unstable
+ String getOriginalTrackingUrl();
+
+ @Private
+ @Unstable
+ void setOriginalTrackingUrl(String url);
/**
* Get the <em>start time</em> of the application.
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Wed Oct 26 06:29:21 2011
@@ -91,6 +91,15 @@ implements ApplicationReport {
}
@Override
+ public String getOriginalTrackingUrl() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasOriginalTrackingUrl()) {
+ return null;
+ }
+ return p.getOriginalTrackingUrl();
+ }
+
+ @Override
public String getName() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasName()) {
@@ -198,6 +207,16 @@ implements ApplicationReport {
}
builder.setTrackingUrl(url);
}
+
+ @Override
+ public void setOriginalTrackingUrl(String url) {
+ maybeInitBuilder();
+ if (url == null) {
+ builder.clearOriginalTrackingUrl();
+ return;
+ }
+ builder.setOriginalTrackingUrl(url);
+ }
@Override
public void setName(String name) {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Wed Oct 26 06:29:21 2011
@@ -162,6 +162,7 @@ message ApplicationReportProto {
optional int64 finishTime = 14;
optional FinalApplicationStatusProto final_application_status = 15;
optional ApplicationResourceUsageReportProto app_resource_Usage = 16;
+ optional string originalTrackingUrl = 17;
}
message NodeIdProto {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Oct 26 06:29:21 2011
@@ -371,6 +371,22 @@ public class YarnConfiguration extends C
public static final int INVALID_CONTAINER_EXIT_STATUS = -1000;
public static final int ABORTED_CONTAINER_EXIT_STATUS = -100;
+
+ ////////////////////////////////
+ // Web Proxy Configs
+ ////////////////////////////////
+ public static final String PROXY_PREFIX = "yarn.web-proxy.";
+
+ /** The kerberos principal for the proxy.*/
+ public static final String PROXY_PRINCIPAL =
+ PROXY_PREFIX + "principal";
+
+ /** Keytab for Proxy.*/
+ public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab";
+
+ /** The address for the web proxy.*/
+ public static final String PROXY_ADDRESS =
+ PROXY_PREFIX + "address";
/**
* YARN Service Level Authorization
@@ -406,15 +422,27 @@ public class YarnConfiguration extends C
}
}
- public static String getRMWebAppURL(Configuration conf) {
+ public static String getProxyHostAndPort(Configuration conf) {
+ String addr = conf.get(PROXY_ADDRESS);
+ if(addr == null || addr.isEmpty()) {
+ addr = getRMWebAppHostAndPort(conf);
+ }
+ return addr;
+ }
+
+ public static String getRMWebAppHostAndPort(Configuration conf) {
String addr = conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
Iterator<String> it = ADDR_SPLITTER.split(addr).iterator();
it.next(); // ignore the bind host
String port = it.next();
// Use apps manager address to figure out the host for webapp
addr = conf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS);
String host = ADDR_SPLITTER.split(addr).iterator().next();
- return JOINER.join("http://", host, ":", port);
+ return JOINER.join(host, ":", port);
+ }
+
+ public static String getRMWebAppURL(Configuration conf) {
+ return JOINER.join("http://", getRMWebAppHostAndPort(conf));
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Wed Oct 26 06:29:21 2011
@@ -281,7 +281,8 @@ public class BuilderUtils {
ApplicationId applicationId, String user, String queue, String name,
String host, int rpcPort, String clientToken, YarnApplicationState state,
String diagnostics, String url, long startTime, long finishTime,
- FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources) {
+ FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources,
+ String origTrackingUrl) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@@ -298,6 +299,7 @@ public class BuilderUtils {
report.setFinishTime(finishTime);
report.setFinalApplicationStatus(finalStatus);
report.setApplicationResourceUsageReport(appResources);
+ report.setOriginalTrackingUrl(origTrackingUrl);
return report;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java Wed Oct 26 06:29:21 2011
@@ -154,15 +154,24 @@ public final class StringHelper {
if (part.startsWith("#") || isAbsUrl(part)) {
sb.append(part);
} else {
- sb.append('/').append(pathPrefix).append('/').append(part);
+ uappend(sb, pathPrefix);
+ uappend(sb, part);
}
} else {
- sb.append('/').append(part);
+ uappend(sb, part);
}
}
return sb.toString();
}
-
+
+ private static void uappend(StringBuilder sb, String part) {
+ if((sb.length() <= 0 || sb.charAt(sb.length() - 1) != '/')
+ && !part.startsWith("/")) {
+ sb.append('/');
+ }
+ sb.append(part);
+ }
+
public static String percent(double value) {
return String.format("%.2f", value * 100);
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/View.java Wed Oct 26 06:29:21 2011
@@ -32,6 +32,7 @@ import javax.servlet.http.HttpServletRes
import static org.apache.hadoop.yarn.util.StringHelper.*;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -175,8 +176,20 @@ public abstract class View implements Pa
moreParams().put(key, value);
}
+ public String root() {
+ String root = System.getenv(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV);
+ if(root == null || root.isEmpty()) {
+ root = "/";
+ }
+ return root;
+ }
+
public String prefix() {
- return context().rc.prefix;
+ if(context().rc.prefix == null) {
+ return root();
+ } else {
+ return ujoin(root(), context().rc.prefix);
+ }
}
public void setTitle(String title) {
@@ -193,6 +206,16 @@ public abstract class View implements Pa
* @param parts components to join
* @return an url string
*/
+ public String root_url(String... parts) {
+ return ujoin(root(), parts);
+ }
+
+
+ /**
+ * Create an url from url components
+ * @param parts components to join
+ * @return an url string
+ */
public String url(String... parts) {
return ujoin(prefix(), parts);
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Wed Oct 26 06:29:21 2011
@@ -18,23 +18,27 @@
package org.apache.hadoop.yarn.webapp;
-import static com.google.common.base.Preconditions.*;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.servlet.GuiceFilter;
+import static com.google.common.base.Preconditions.checkNotNull;
import java.net.ConnectException;
import java.net.URL;
-import org.apache.commons.lang.StringUtils;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import javax.servlet.http.HttpServlet;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceFilter;
+
/**
* Helpers to create an embedded webapp.
*
@@ -59,6 +63,12 @@ public class WebApps {
static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
public static class Builder<T> {
+ static class ServletStruct {
+ public Class<? extends HttpServlet> clazz;
+ public String name;
+ public String spec;
+ }
+
final String name;
final Class<T> api;
final T application;
@@ -67,6 +77,8 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
boolean devMode = false;
+ private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
+ private final HashMap<String, Object> attributes = new HashMap<String, Object>();
Builder(String name, Class<T> api, T application) {
this.name = name;
@@ -93,6 +105,21 @@ public class WebApps {
return this;
}
+ public Builder<T> withAttribute(String key, Object value) {
+ attributes.put(key, value);
+ return this;
+ }
+
+ public Builder<T> withServlet(String name, String pathSpec,
+ Class<? extends HttpServlet> servlet) {
+ ServletStruct struct = new ServletStruct();
+ struct.clazz = servlet;
+ struct.name = name;
+ struct.spec = pathSpec;
+ servlets.add(struct);
+ return this;
+ }
+
public Builder<T> with(Configuration conf) {
this.conf = conf;
return this;
@@ -152,6 +179,12 @@ public class WebApps {
HttpServer server =
new HttpServer(name, bindAddress, port, findPort, conf,
webapp.getServePathSpecs());
+ for(ServletStruct struct: servlets) {
+ server.addServlet(struct.name, struct.spec, struct.clazz);
+ }
+ for(Map.Entry<String, Object> entry : attributes.entrySet()) {
+ server.setAttribute(entry.getKey(), entry.getValue());
+ }
server.addGlobalFilter("guice", GuiceFilter.class.getName(), null);
webapp.setConf(conf);
webapp.setHttpServer(server);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/ErrorPage.java Wed Oct 26 06:29:21 2011
@@ -34,7 +34,7 @@ public class ErrorPage extends HtmlPage
String title = "Sorry, got error "+ status();
html.
title(title).
- link("/static/yarn.css").
+ link(root_url("static","yarn.css")).
_(JQueryUI.class). // an embedded sub-view
style("#msg { margin: 1em auto; width: 88%; }",
"#msg h1 { padding: 0.2em 1.5em; font: bold 1.3em serif; }").
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java Wed Oct 26 06:29:21 2011
@@ -81,12 +81,12 @@ public class JQueryUI extends HtmlBlock
html.
link(join("https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.9/themes/",
getTheme(), "/jquery-ui.css")).
- link("/static/dt-1.7.5/css/jui-dt.css").
+ link(root_url("static/dt-1.7.5/css/jui-dt.css")).
script("https://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js").
script("https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.9/jquery-ui.min.js").
- script("/static/dt-1.7.5/js/jquery.dataTables.min.js").
- script("/static/yarn.dt.plugins.js").
- script("/static/themeswitcher.js").
+ script(root_url("static/dt-1.7.5/js/jquery.dataTables.min.js")).
+ script(root_url("static/yarn.dt.plugins.js")).
+ script(root_url("static/themeswitcher.js")).
style("#jsnotice { padding: 0.2em; text-align: center; }",
".ui-progressbar { height: 1em; min-width: 5em }"); // required
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnCssLayout.java Wed Oct 26 06:29:21 2011
@@ -35,7 +35,7 @@ public class TwoColumnCssLayout extends
preHead(html);
html.
title($("title")).
- link("/static/yarn.css").
+ link(root_url("static","yarn.css")).
style(".main { min-height: 100%; height: auto !important; height: 100%;",
" margin: 0 auto -4em; border: 0; }",
".footer, .push { height: 4em; clear: both; border: 0 }",
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java Wed Oct 26 06:29:21 2011
@@ -41,7 +41,7 @@ public class TwoColumnLayout extends Htm
preHead(html);
html.
title($(TITLE)).
- link("/static/yarn.css").
+ link(root_url("static","yarn.css")).
style("#layout { height: 100%; }",
"#layout thead td { height: 3em; }",
"#layout #navcell { width: 11em; padding: 0 1em; }",
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java Wed Oct 26 06:29:21 2011
@@ -89,6 +89,7 @@ public class MockApps {
@Override public YarnApplicationState getYarnApplicationState() { return state; }
@Override public String getQueue() { return queue; }
@Override public String getTrackingUrl() { return ""; }
+ @Override public String getOriginalTrackingUrl() { return ""; }
@Override public FinalApplicationStatus getFinalApplicationStatus() { return finishState; }
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
@@ -103,6 +104,7 @@ public class MockApps {
// TODO Auto-generated method stub
}
+ @Override public void setOriginalTrackingUrl(String url) { }
@Override
public void setApplicationResourceUsageReport(ApplicationResourceUsageReport appResources) {
this.appUsageReport = appResources;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml Wed Oct 26 06:29:21 2011
@@ -376,4 +376,27 @@
<name>mapreduce.job.hdfs-servers</name>
<value>${fs.default.name}</value>
</property>
+
+ <!-- WebAppProxy Configuration-->
+
+ <property>
+ <description>The kerberos principal for the proxy, if the proxy is not
+ running as part of the RM.</description>
+ <name>yarn.web-proxy.principal</name>
+ <value/>
+ </property>
+
+ <property>
+ <description>Keytab for WebAppProxy, if the proxy is not running as part of
+ the RM.</description>
+ <name>yarn.web-proxy.keytab</name>
+ </property>
+
+ <property>
+ <description>The address for the web proxy as HOST:PORT, if this is not
+ given or if it matches yarn.resourcemanager.address then the proxy will
+ run as part of the RM</description>
+ <name>yarn.web-proxy.address</name>
+ <value/>
+ </property>
</configuration>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Wed Oct 26 06:29:21 2011
@@ -34,6 +34,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Oct 26 06:29:21 2011
@@ -66,11 +66,16 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.hadoop.yarn.webapp.WebApps.Builder;
/**
* The ResourceManager is the main class that is a set of components.
@@ -406,11 +411,18 @@ public class ResourceManager extends Com
}
protected void startWepApp() {
- webApp = WebApps.$for("cluster", masterService).at(
- this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS)).
- start(new RMWebApp(this));
-
+ Builder<ApplicationMasterService> builder =
+ WebApps.$for("cluster", masterService).at(
+ this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS));
+ if(YarnConfiguration.getRMWebAppHostAndPort(conf).
+ equals(YarnConfiguration.getProxyHostAndPort(conf))) {
+ AppReportFetcher fetcher = new AppReportFetcher(conf, getClientRMService());
+ builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
+ ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
+ builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
+ }
+ webApp = builder.start(new RMWebApp(this));
}
@Override
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Wed Oct 26 06:29:21 2011
@@ -187,6 +187,8 @@ public class AMLauncher implements Runna
throws IOException {
Map<String, String> environment = container.getEnvironment();
+ environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
+ application.getWebProxyBase());
// Set the AppAttemptId, containerId, NMHTTPAdress, AppSubmitTime to be
// consumable by the AM.
environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, container
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Wed Oct 26 06:29:21 2011
@@ -331,11 +331,13 @@ public class RMAppImpl implements RMApp
String clientToken = "N/A";
String trackingUrl = "N/A";
String host = "N/A";
+ String origTrackingUrl = "N/A";
int rpcPort = -1;
ApplicationResourceUsageReport appUsageReport = null;
FinalApplicationStatus finishState = getFinalApplicationStatus();
if (this.currentAttempt != null) {
trackingUrl = this.currentAttempt.getTrackingUrl();
+ origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
clientToken = this.currentAttempt.getClientToken();
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
@@ -345,7 +347,8 @@ public class RMAppImpl implements RMApp
this.queue, this.name, host, rpcPort, clientToken,
createApplicationState(this.stateMachine.getCurrentState()),
this.diagnostics.toString(), trackingUrl,
- this.startTime, this.finishTime, finishState, appUsageReport);
+ this.startTime, this.finishTime, finishState, appUsageReport,
+ origTrackingUrl);
} finally {
this.readLock.unlock();
}
@@ -381,7 +384,7 @@ public class RMAppImpl implements RMApp
@Override
public String getTrackingUrl() {
this.readLock.lock();
-
+
try {
if (this.currentAttempt != null) {
return this.currentAttempt.getTrackingUrl();
@@ -439,7 +442,7 @@ public class RMAppImpl implements RMApp
RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId,
clientTokenStr, rmContext, scheduler, masterService,
- submissionContext);
+ submissionContext, YarnConfiguration.getProxyHostAndPort(conf));
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
handler.handle(
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Wed Oct 26 06:29:21 2011
@@ -72,6 +72,22 @@ public interface RMAppAttempt extends Ev
String getTrackingUrl();
/**
+ * The original url at which the status of the application attempt can be
+ * accessed. This url is not fronted by a proxy. This is only intended to be
+ * used by the proxy.
+ * @return the url at which the status of the attempt can be accessed and is
+ * not fronted by a proxy.
+ */
+ String getOriginalTrackingUrl();
+
+ /**
+ * The base to be prepended to web URLs that are not relative, and the user
+ * has been checked.
+ * @return the base URL to be prepended to web URLs that are not relative.
+ */
+ String getWebProxyBase();
+
+ /**
* The token required by the clients to talk to the application attempt
* @return the token required by the clients to talk to the application attempt
*/
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Wed Oct 26 06:29:21 2011
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -67,6 +69,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
@@ -114,12 +117,16 @@ public class RMAppAttemptImpl implements
private float progress = 0;
private String host = "N/A";
private int rpcPort;
- private String trackingUrl = "N/A";
+ private String origTrackingUrl = "N/A";
+ private String proxiedTrackingUrl = "N/A";
+
// Set to null initially. Will eventually get set
// if an RMAppAttemptUnregistrationEvent occurs
private FinalApplicationStatus finalStatus = null;
private final StringBuilder diagnostics = new StringBuilder();
+ private final String proxy;
+
private static final StateMachineFactory<RMAppAttemptImpl,
RMAppAttemptState,
RMAppAttemptEventType,
@@ -250,8 +257,10 @@ public class RMAppAttemptImpl implements
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
String clientToken, RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
- ApplicationSubmissionContext submissionContext) {
+ ApplicationSubmissionContext submissionContext,
+ String proxy) {
+ this.proxy = proxy;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
@@ -322,9 +331,46 @@ public class RMAppAttemptImpl implements
@Override
public String getTrackingUrl() {
this.readLock.lock();
-
try {
- return this.trackingUrl;
+ return this.proxiedTrackingUrl;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public String getOriginalTrackingUrl() {
+ this.readLock.lock();
+ try {
+ return this.origTrackingUrl;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public String getWebProxyBase() {
+ this.readLock.lock();
+ try {
+ return ProxyUriUtils.getPath(applicationAttemptId.getApplicationId());
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ private String generateProxyUriWithoutScheme(
+ final String trackingUriWithoutScheme) {
+ this.readLock.lock();
+ try {
+ URI trackingUri = ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
+ URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+ applicationAttemptId.getApplicationId());
+ //We need to strip off the scheme to have it match what was there before
+ return result.toASCIIString().substring(7);
+ } catch (URISyntaxException e) {
+ LOG.warn("Could not proxify "+trackingUriWithoutScheme,e);
+ return trackingUriWithoutScheme;
} finally {
this.readLock.unlock();
}
@@ -691,7 +737,9 @@ public class RMAppAttemptImpl implements
= (RMAppAttemptRegistrationEvent) event;
appAttempt.host = registrationEvent.getHost();
appAttempt.rpcPort = registrationEvent.getRpcport();
- appAttempt.trackingUrl = registrationEvent.getTrackingurl();
+ appAttempt.origTrackingUrl = registrationEvent.getTrackingurl();
+ appAttempt.proxiedTrackingUrl =
+ appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
// Let the app know
appAttempt.eventHandler.handle(new RMAppEvent(appAttempt
@@ -787,7 +835,9 @@ public class RMAppAttemptImpl implements
RMAppAttemptUnregistrationEvent unregisterEvent
= (RMAppAttemptUnregistrationEvent) event;
appAttempt.diagnostics.append(unregisterEvent.getDiagnostics());
- appAttempt.trackingUrl = unregisterEvent.getTrackingUrl();
+ appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl();
+ appAttempt.proxiedTrackingUrl =
+ appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
// Tell the app and the scheduler
@@ -837,7 +887,8 @@ public class RMAppAttemptImpl implements
* NOTE: don't set trackingUrl to 'null'. That will cause null-pointer exceptions
* in the generated proto code.
*/
- appAttempt.trackingUrl = "";
+ appAttempt.origTrackingUrl = "";
+ appAttempt.proxiedTrackingUrl = "";
new FinalTransition(RMAppAttemptState.FAILED).transition(
appAttempt, containerFinishedEvent);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1189036&r1=1189035&r2=1189036&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Wed Oct 26 06:29:21 2011
@@ -178,7 +178,7 @@ public class TestRMAppAttemptTransitions
application = mock(RMApp.class);
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, null, rmContext, scheduler,
- masterService, submissionContext);
+ masterService, submissionContext, null);
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
@@ -328,7 +328,9 @@ public class TestRMAppAttemptTransitions
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
- assertEquals(trackingUrl, applicationAttempt.getTrackingUrl());
+ assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
+ getApplicationId()+"/", applicationAttempt.getTrackingUrl());
// TODO - need to add more checks relevant to this state
}
@@ -343,7 +345,9 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINISHED,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getTrackingUrl());
+ assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
+ getApplicationId()+"/", applicationAttempt.getTrackingUrl());
assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1189036&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Wed Oct 26 06:29:21 2011
@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project>
+ <parent>
+ <artifactId>hadoop-yarn-server</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>0.24.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ <version>0.24.0-SNAPSHOT</version>
+ <name>hadoop-yarn-server-web-proxy</name>
+
+ <properties>
+ <install.file>${project.artifact.file}</install.file>
+ <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <scope>compile</scope>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+
+ <!-- Publish tests jar -->
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>test-compile</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java?rev=1189036&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java Wed Oct 26 06:29:21 2011
@@ -0,0 +1,93 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.webproxy;
+
+import java.net.InetSocketAddress;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+
+/**
+ * This class abstracts away how ApplicationReports are fetched.
+ */
+public class AppReportFetcher {
+ private static final Log LOG = LogFactory.getLog(AppReportFetcher.class);
+ private final Configuration conf;
+ private final ClientRMProtocol applicationsManager;
+ private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+
+ /**
+ * Create a new Connection to the RM to fetch Application reports.
+ * @param conf the conf to use to know where the RM is.
+ */
+ public AppReportFetcher(Configuration conf) {
+ this.conf = conf;
+ YarnRPC rpc = YarnRPC.create(this.conf);
+ InetSocketAddress rmAddress =
+ NetUtils.createSocketAddr(this.conf.get(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
+ LOG.info("Connecting to ResourceManager at " + rmAddress);
+ applicationsManager =
+ (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,
+ rmAddress, this.conf);
+ LOG.info("Connected to ResourceManager at " + rmAddress);
+ }
+
+ /**
+ * Just call directly into the applicationsManager given instead of creating
+ * a remote connection to it. This is mostly for when the Proxy is running
+ * as part of the RM already.
+ * @param conf the configuration to use
+ * @param applicationsManager what to use to get the RM reports.
+ */
+ public AppReportFetcher(Configuration conf, ClientRMProtocol applicationsManager) {
+ this.conf = conf;
+ this.applicationsManager = applicationsManager;
+ }
+
+ /**
+ * Get a report for the specified app.
+ * @param appId the id of the application to get.
+ * @return the ApplicationReport for that app.
+ * @throws YarnRemoteException on any error.
+ */
+ public ApplicationReport getApplicationReport(ApplicationId appId)
+ throws YarnRemoteException {
+ GetApplicationReportRequest request = recordFactory
+ .newRecordInstance(GetApplicationReportRequest.class);
+ request.setApplicationId(appId);
+
+ GetApplicationReportResponse response = applicationsManager
+ .getApplicationReport(request);
+ return response.getApplicationReport();
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1189036&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java Wed Oct 26 06:29:21 2011
@@ -0,0 +1,143 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.webproxy;
+
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+public class ProxyUriUtils {
+ @SuppressWarnings("unused")
+ private static final Log LOG = LogFactory.getLog(ProxyUriUtils.class);
+
+ /**Name of the servlet to use when registering the proxy servlet. */
+ public static final String PROXY_SERVLET_NAME = "proxy";
+ /**Base path where the proxy servlet will handle requests.*/
+ public static final String PROXY_BASE = "/proxy/";
+ /**Path Specification for the proxy servlet.*/
+ public static final String PROXY_PATH_SPEC = PROXY_BASE+"*";
+ /**Query Parameter indicating that the URI was approved.*/
+ public static final String PROXY_APPROVAL_PARAM = "proxyapproved";
+
+ private static String uriEncode(Object o) {
+ try {
+ assert (o != null) : "o canot be null";
+ return URLEncoder.encode(o.toString(), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ //This should never happen
+ throw new RuntimeException("UTF-8 is not supported by this system?", e);
+ }
+ }
+
+ /**
+ * Get the proxied path for an application.
+ * @param id the application id to use.
+ * @return the base path to that application through the proxy.
+ */
+ public static String getPath(ApplicationId id) {
+ if(id == null) {
+ throw new IllegalArgumentException("Application id cannot be null ");
+ }
+ return ujoin(PROXY_BASE, uriEncode(id));
+ }
+
+ /**
+ * Get the proxied path for an application.
+ * @param id the application id to use.
+ * @param path the rest of the path to the application.
+ * @return the base path to that application through the proxy.
+ */
+ public static String getPath(ApplicationId id, String path) {
+ if(path == null) {
+ return getPath(id);
+ } else {
+ return ujoin(getPath(id), path);
+ }
+ }
+
+ /**
+ * Get the proxied path for an application
+ * @param id the id of the application
+ * @param path the path of the application.
+ * @param query the query parameters
+ * @param approved true if the user has approved accessing this app.
+ * @return the proxied path for this app.
+ */
+ public static String getPathAndQuery(ApplicationId id, String path,
+ String query, boolean approved) {
+ StringBuilder newp = new StringBuilder();
+ newp.append(getPath(id, path));
+ boolean first = appendQuery(newp, query, true);
+ if(approved) {
+ first = appendQuery(newp, PROXY_APPROVAL_PARAM+"=true", first);
+ }
+ return newp.toString();
+ }
+
+ private static boolean appendQuery(StringBuilder builder, String query,
+ boolean first) {
+ if(query != null && !query.isEmpty()) {
+ if(first && !query.startsWith("?")) {
+ builder.append('?');
+ }
+ if(!first && !query.startsWith("&")) {
+ builder.append('&');
+ }
+ builder.append(query);
+ return false;
+ }
+ return first;
+ }
+
+ /**
+ * Get a proxied URI for the original URI.
+ * @param originalUri the original URI to go through the proxy
+ * @param proxyUri the URI of the proxy itself, scheme, host and port are used.
+ * @param id the id of the application
+ * @return the proxied URI
+ */
+ public static URI getProxyUri(URI originalUri, URI proxyUri,
+ ApplicationId id) {
+ try {
+ String path = getPath(id, originalUri.getPath());
+ return new URI(proxyUri.getScheme(), proxyUri.getAuthority(), path,
+ originalUri.getQuery(), originalUri.getFragment());
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Could not proxify "+originalUri,e);
+ }
+ }
+
+ /**
+ * Create a URI form a no scheme Url, such as is returned by the AM.
+ * @param noSchemeUrl the URL formate returned by an AM
+ * @return a URI with an http scheme
+ * @throws URISyntaxException if the url is not formatted correctly.
+ */
+ public static URI getUriFromAMUrl(String noSchemeUrl)
+ throws URISyntaxException {
+ return new URI("http://"+noSchemeUrl);
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1189036&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Wed Oct 26 06:29:21 2011
@@ -0,0 +1,111 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.webproxy;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.AbstractService;
+
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+
+public class WebAppProxy extends AbstractService {
+ public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher";
+ public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled";
+ private static final Log LOG = LogFactory.getLog(WebAppProxy.class);
+
+ private HttpServer proxyServer = null;
+ private String bindAddress = null;
+ private int port = 0;
+ private AccessControlList acl = null;
+ private AppReportFetcher fetcher = null;
+ private boolean isSecurityEnabled = false;
+
+ public WebAppProxy() {
+ super(WebAppProxy.class.getName());
+ }
+
+ @Override
+ public void init(Configuration conf) {
+ String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
+ if (auth == null || "simple".equals(auth)) {
+ isSecurityEnabled = false;
+ } else if ("kerberos".equals(auth)) {
+ isSecurityEnabled = true;
+ } else {
+ LOG.warn("Unrecongized attribute value for " +
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
+ " of " + auth);
+ }
+
+ fetcher = new AppReportFetcher(conf);
+ bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
+ if(bindAddress == null || bindAddress.isEmpty()) {
+ throw new YarnException(YarnConfiguration.PROXY_ADDRESS +
+ " is not set so the proxy will not run.");
+ }
+ LOG.info("Instantiating Proxy at " + bindAddress);
+ String[] parts = StringUtils.split(bindAddress, ':');
+ port = 0;
+ if (parts.length == 2) {
+ bindAddress = parts[0];
+ port = Integer.parseInt(parts[1]);
+ }
+ acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ super.init(conf);
+ }
+
+ @Override
+ public void start() {
+ try {
+ proxyServer = new HttpServer("proxy", bindAddress, port,
+ port == 0, getConfig(), acl);
+ proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
+ ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
+ proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
+ proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
+ proxyServer.start();
+ } catch (IOException e) {
+ LOG.fatal("Could not start proxy web server",e);
+ throw new YarnException("Could not start proxy web server",e);
+ }
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ if(proxyServer != null) {
+ try {
+ proxyServer.stop();
+ } catch (Exception e) {
+ LOG.fatal("Error stopping proxy web server", e);
+ throw new YarnException("Error stopping proxy web server",e);
+ }
+ }
+ super.stop();
+ }
+}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java?rev=1189036&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java Wed Oct 26 06:29:21 2011
@@ -0,0 +1,83 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.webproxy;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.CompositeService;
+
+/**
+ * ProxyServer will sit in between the end user and AppMaster
+ * web interfaces.
+ */
+public class WebAppProxyServer extends CompositeService {
+ private static final Log LOG = LogFactory.getLog(WebAppProxyServer.class);
+
+ private WebAppProxy proxy = null;
+
+ public WebAppProxyServer() {
+ super(WebAppProxyServer.class.getName());
+ }
+
+ @Override
+ public synchronized void init(Configuration conf) {
+ Configuration config = new YarnConfiguration(conf);
+ try {
+ doSecureLogin(conf);
+ } catch(IOException ie) {
+ throw new YarnException("Proxy Server Failed to login", ie);
+ }
+ proxy = new WebAppProxy();
+ addService(proxy);
+ super.init(config);
+ }
+
+ /**
+ * Log in as the Kerberose principal designated for the proxy
+ * @param conf the configuration holding this information in it.
+ * @throws IOException on any error.
+ */
+ protected void doSecureLogin(Configuration conf) throws IOException {
+ SecurityUtil.login(conf, YarnConfiguration.PROXY_KEYTAB,
+ YarnConfiguration.PROXY_PRINCIPAL);
+ }
+
+ public static void main(String[] args) {
+ StringUtils.startupShutdownMessage(WebAppProxyServer.class, args, LOG);
+ try {
+ WebAppProxyServer proxy = new WebAppProxyServer();
+ Runtime.getRuntime().addShutdownHook(
+ new CompositeServiceShutdownHook(proxy));
+ YarnConfiguration conf = new YarnConfiguration();
+ proxy.init(conf);
+ proxy.start();
+ } catch (Throwable t) {
+ LOG.fatal("Error starting Proxy server", t);
+ System.exit(-1);
+ }
+ }
+
+}