You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2012/03/04 22:16:46 UTC
svn commit: r1296881 - in
/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/
hadoop-yarn/hadoop-yarn-server/h...
Author: acmurthy
Date: Sun Mar 4 21:16:45 2012
New Revision: 1296881
URL: http://svn.apache.org/viewvc?rev=1296881&view=rev
Log:
Merge -c 1296878 from trunk to branch-0.23.2 to fix MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured hostname or IP.
Modified:
hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt?rev=1296881&r1=1296880&r2=1296881&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/CHANGES.txt Sun Mar 4 21:16:45 2012
@@ -124,6 +124,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3929. Fixed output of 'bin/mapred queue -showacl' command to
clarify ACLs for users. (John George via acmurthy)
+ MAPREDUCE-3960. Fix web-proxy to forward request to AM with configured
+ hostname or IP. (tgraves via acmurthy)
+
Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1296881&r1=1296880&r2=1296881&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Sun Mar 4 21:16:45 2012
@@ -432,6 +432,10 @@ public class ResourceManager extends Com
builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
+ String proxy = YarnConfiguration.getProxyHostAndPort(conf);
+ String[] proxyParts = proxy.split(":");
+ builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
+
}
webApp = builder.start(new RMWebApp(this));
}
Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1296881&r1=1296880&r2=1296881&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original)
+++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Sun Mar 4 21:16:45 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.CommonConfig
public class WebAppProxy extends AbstractService {
public static final String FETCHER_ATTRIBUTE= "AppUrlFetcher";
public static final String IS_SECURITY_ENABLED_ATTRIBUTE = "IsSecurityEnabled";
+ public static final String PROXY_HOST_ATTRIBUTE = "proxyHost";
private static final Log LOG = LogFactory.getLog(WebAppProxy.class);
private HttpServer proxyServer = null;
@@ -43,6 +44,7 @@ public class WebAppProxy extends Abstrac
private AccessControlList acl = null;
private AppReportFetcher fetcher = null;
private boolean isSecurityEnabled = false;
+ private String proxyHost = null;
public WebAppProxy() {
super(WebAppProxy.class.getName());
@@ -60,6 +62,9 @@ public class WebAppProxy extends Abstrac
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
" of " + auth);
}
+ String proxy = YarnConfiguration.getProxyHostAndPort(conf);
+ String[] proxyParts = proxy.split(":");
+ proxyHost = proxyParts[0];
fetcher = new AppReportFetcher(conf);
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
@@ -88,6 +93,7 @@ public class WebAppProxy extends Abstrac
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
+ proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
} catch (IOException e) {
LOG.fatal("Could not start proxy web server",e);
Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1296881&r1=1296880&r2=1296881&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java (original)
+++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java Sun Mar 4 21:16:45 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
@@ -38,6 +39,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
@@ -123,13 +125,23 @@ public class WebAppProxyServlet extends
* @throws IOException on any error.
*/
private static void proxyLink(HttpServletRequest req,
- HttpServletResponse resp, URI link,Cookie c) throws IOException {
+ HttpServletResponse resp, URI link, Cookie c, String proxyHost)
+ throws IOException {
org.apache.commons.httpclient.URI uri =
new org.apache.commons.httpclient.URI(link.toString(), false);
HttpClientParams params = new HttpClientParams();
params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true);
HttpClient client = new HttpClient(params);
+ // Make sure we send the request from the proxy address in the config
+ // since that is what the AM filter checks against. IP aliasing or
+ // similar could cause issues otherwise.
+ HostConfiguration config = new HostConfiguration();
+ InetAddress localAddress = InetAddress.getByName(proxyHost);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("local InetAddress for proxy host: " + localAddress.toString());
+ }
+ config.setLocalAddress(localAddress);
HttpMethod method = new GetMethod(uri.getEscapedURI());
@SuppressWarnings("unchecked")
@@ -150,7 +162,7 @@ public class WebAppProxyServlet extends
}
OutputStream out = resp.getOutputStream();
try {
- resp.setStatus(client.executeMethod(method));
+ resp.setStatus(client.executeMethod(config, method));
for(Header header : method.getResponseHeaders()) {
resp.setHeader(header.getName(), header.getValue());
}
@@ -189,6 +201,11 @@ public class WebAppProxyServlet extends
.getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)).getApplicationReport(id);
}
+ private String getProxyHost() throws IOException {
+ return ((String) getServletContext()
+ .getAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE));
+ }
+
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws IOException{
@@ -275,7 +292,7 @@ public class WebAppProxyServlet extends
if(userWasWarned && userApproved) {
c = makeCheckCookie(id, true);
}
- proxyLink(req, resp, toFetch, c);
+ proxyLink(req, resp, toFetch, c, getProxyHost());
} catch(URISyntaxException e) {
throw new IOException(e);
Modified: hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java?rev=1296881&r1=1296880&r2=1296881&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java (original)
+++ hadoop/common/branches/branch-0.23.2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmIpFilter.java Sun Mar 4 21:16:45 2012
@@ -64,6 +64,9 @@ public class AmIpFilter implements Filte
try {
proxyAddresses = new HashSet<String>();
for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("proxy address is: " + add.getHostAddress());
+ }
proxyAddresses.add(add.getHostAddress());
}
lastUpdate = now;
@@ -89,6 +92,9 @@ public class AmIpFilter implements Filte
HttpServletRequest httpReq = (HttpServletRequest)req;
HttpServletResponse httpResp = (HttpServletResponse)resp;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remote address for request is: " + httpReq.getRemoteAddr());
+ }
if(!getProxyAddresses().contains(httpReq.getRemoteAddr())) {
String redirectUrl = httpResp.encodeRedirectURL(proxyUriBase +
httpReq.getRequestURI());