You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/12/17 00:42:57 UTC
svn commit: r1215347 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/
hadoop-yarn/hadoop-y...
Author: vinodkv
Date: Fri Dec 16 23:42:56 2011
New Revision: 1215347
URL: http://svn.apache.org/viewvc?rev=1215347&view=rev
Log:
MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even before the job started, so that it works properly with oozie throughout the job execution. Contributed by Robert Joseph Evans.
svn merge -c 1215345 --ignore-ancestry ../../trunk/
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1215347&r1=1215346&r2=1215347&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Dec 16 23:42:56 2011
@@ -275,6 +275,10 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3366. Mapreduce component should use consistent directory structure
layout as HDFS/common (Eric Yang via mahadev)
+ MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even
+ before the job started, so that it works properly with oozie throughout
+ the job execution. (Robert Joseph Evans via vinodkv)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1215347&r1=1215346&r2=1215347&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Dec 16 23:42:56 2011
@@ -273,6 +273,8 @@ public class RMAppAttemptImpl implements
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
+ this.proxiedTrackingUrl = generateProxyUriWithoutScheme();
+
this.stateMachine = stateMachineFactory.make(this);
}
@@ -358,11 +360,16 @@ public class RMAppAttemptImpl implements
}
}
+ private String generateProxyUriWithoutScheme() {
+ return generateProxyUriWithoutScheme(null);
+ }
+
private String generateProxyUriWithoutScheme(
final String trackingUriWithoutScheme) {
this.readLock.lock();
try {
- URI trackingUri = ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
+ URI trackingUri = trackingUriWithoutScheme == null ? null :
+ ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
applicationAttemptId.getApplicationId());
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1215347&r1=1215346&r2=1215347&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java Fri Dec 16 23:42:56 2011
@@ -80,8 +80,10 @@ public class AppInfo {
if (app != null) {
String trackingUrl = app.getTrackingUrl();
+ this.state = app.getState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
- || "N/A".equalsIgnoreCase(trackingUrl);
+ || RMAppState.NEW == this.state || RMAppState.SUBMITTED == this.state
+ || RMAppState.ACCEPTED == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
.getFinishTime() == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady) {
@@ -95,7 +97,6 @@ public class AppInfo {
this.user = app.getUser().toString();
this.name = app.getName().toString();
this.queue = app.getQueue().toString();
- this.state = app.getState();
this.progress = app.getProgress() * 100;
this.diagnostics = app.getDiagnostics().toString();
if (diagnostics == null || diagnostics.isEmpty()) {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1215347&r1=1215346&r2=1215347&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Fri Dec 16 23:42:56 2011
@@ -17,8 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.*;
import static org.mockito.Matchers.*;
import static org.mockito.Mockito.*;
@@ -203,6 +202,8 @@ public class TestRMAppAttemptTransitions
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
+ assertNotNull(applicationAttempt.getTrackingUrl());
+ assertFalse("N/A".equals(applicationAttempt.getTrackingUrl()));
}
/**
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1215347&r1=1215346&r2=1215347&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java Fri Dec 16 23:42:56 2011
@@ -114,7 +114,8 @@ public class ProxyUriUtils {
/**
* Get a proxied URI for the original URI.
- * @param originalUri the original URI to go through the proxy
+ * @param originalUri the original URI to go through the proxy, or null if
+ * a default path "/" can be used.
* @param proxyUri the URI of the proxy itself, scheme, host and port are used.
* @param id the id of the application
* @return the proxied URI
@@ -122,9 +123,10 @@ public class ProxyUriUtils {
public static URI getProxyUri(URI originalUri, URI proxyUri,
ApplicationId id) {
try {
- String path = getPath(id, originalUri.getPath());
+ String path = getPath(id, originalUri == null ? "/" : originalUri.getPath());
return new URI(proxyUri.getScheme(), proxyUri.getAuthority(), path,
- originalUri.getQuery(), originalUri.getFragment());
+ originalUri == null ? null : originalUri.getQuery(),
+ originalUri == null ? null : originalUri.getFragment());
} catch (URISyntaxException e) {
throw new RuntimeException("Could not proxify "+originalUri,e);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java?rev=1215347&r1=1215346&r2=1215347&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java Fri Dec 16 23:42:56 2011
@@ -23,44 +23,16 @@ import static org.junit.Assert.*;
import java.net.URI;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
public class TestProxyUriUtils {
- public static class TestAppId extends ApplicationId {
- private long timestamp;
- private int id;
-
- public TestAppId(int id, long timestamp) {
- setId(id);
- setClusterTimestamp(timestamp);
- }
- @Override
- public int getId() {
- return id;
- }
-
- @Override
- public void setId(int id) {
- this.id = id;
- }
-
- @Override
- public long getClusterTimestamp() {
- return timestamp;
- }
-
- @Override
- public void setClusterTimestamp(long clusterTimestamp) {
- this.timestamp = clusterTimestamp;
- }
- }
-
@Test
public void testGetPathApplicationId() {
assertEquals("/proxy/application_100_0001",
- ProxyUriUtils.getPath(new TestAppId(1, 100l)));
+ ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1)));
assertEquals("/proxy/application_6384623_0005",
- ProxyUriUtils.getPath(new TestAppId(5, 6384623l)));
+ ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5)));
}
@Test(expected = IllegalArgumentException.class)
@@ -71,23 +43,23 @@ public class TestProxyUriUtils {
@Test
public void testGetPathApplicationIdString() {
assertEquals("/proxy/application_6384623_0005",
- ProxyUriUtils.getPath(new TestAppId(5, 6384623l), null));
+ ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null));
assertEquals("/proxy/application_6384623_0005/static/app",
- ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/static/app"));
+ ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app"));
assertEquals("/proxy/application_6384623_0005/",
- ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/"));
+ ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/"));
assertEquals("/proxy/application_6384623_0005/some/path",
- ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "some/path"));
+ ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path"));
}
@Test
public void testGetPathAndQuery() {
assertEquals("/proxy/application_6384623_0005/static/app?foo=bar",
- ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app",
+ ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
"?foo=bar", false));
assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true",
- ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app",
+ ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
"foo=bar&bad=good", true));
}
@@ -95,10 +67,20 @@ public class TestProxyUriUtils {
public void testGetProxyUri() throws Exception {
URI originalUri = new URI("http://host.com/static/foo?bar=bar");
URI proxyUri = new URI("http://proxy.net:8080/");
- TestAppId id = new TestAppId(5, 6384623l);
+ ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/static/foo?bar=bar");
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
+
+ @Test
+ public void testGetProxyUriNull() throws Exception {
+ URI originalUri = null;
+ URI proxyUri = new URI("http://proxy.net:8080/");
+ ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
+ URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/");
+ URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
+ assertEquals(expected, result);
+ }
}