You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2013/04/23 02:14:08 UTC
svn commit: r1470760 - in
/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java...
Author: szetszwo
Date: Tue Apr 23 00:14:06 2013
New Revision: 1470760
URL: http://svn.apache.org/r1470760
Log:
Merge r1470195 through r1470759 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
- copied unchanged from r1470759, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1470195-1470759
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt?rev=1470760&r1=1470759&r2=1470760&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt Tue Apr 23 00:14:06 2013
@@ -327,6 +327,12 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-5163. Update MR App to not use API utility methods for collections
after YARN-441. (Xuan Gong via vinodkv)
+ MAPREDUCE-5066. Added a timeout for the job.end.notification.url. (Ivan
+ Mitic via acmurthy)
+
+ MAPREDUCE-5146. application classloader may be used too early to load
+ classes. (Sangjin Lee via tomwhite)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1470195-1470759
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1470195-1470759
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1470760&r1=1470759&r2=1470760&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Tue Apr 23 00:14:06 2013
@@ -20,10 +20,8 @@ package org.apache.hadoop.mapred;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
@@ -148,6 +146,9 @@ class YarnChild {
// Add tokens to new user so that it may execute its task correctly.
childUGI.addCredentials(credentials);
+ // set job classloader if configured before invoking the task
+ MRApps.setJobClassLoader(job);
+
// Create a final reference to the task for the doAs block
final Task taskFinal = task;
childUGI.doAs(new PrivilegedExceptionAction<Object>() {
@@ -255,9 +256,6 @@ class YarnChild {
final JobConf job = new JobConf(MRJobConfig.JOB_CONF_FILE);
job.setCredentials(credentials);
- // set job classloader if configured
- MRApps.setJobClassLoader(job);
-
String appAttemptIdEnv = System
.getenv(MRJobConfig.APPLICATION_ATTEMPT_ID_ENV);
LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptIdEnv);
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java?rev=1470760&r1=1470759&r2=1470760&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java Tue Apr 23 00:14:06 2013
@@ -27,6 +27,7 @@ import java.net.URL;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.mortbay.log.Log;
@@ -54,6 +55,7 @@ public class JobEndNotifier implements C
protected String proxyConf;
protected int numTries; //Number of tries to attempt notification
protected int waitInterval; //Time (ms) to wait between retrying notification
+ protected int timeout; // Timeout (ms) on the connection and notification
protected URL urlToNotify; //URL to notify read from the config
protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification
@@ -76,6 +78,9 @@ public class JobEndNotifier implements C
);
waitInterval = (waitInterval < 0) ? 5000 : waitInterval;
+ timeout = conf.getInt(JobContext.MR_JOB_END_NOTIFICATION_TIMEOUT,
+ JobContext.DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT);
+
userUrl = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL);
proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);
@@ -112,8 +117,7 @@ public class JobEndNotifier implements C
}
/**
- * Notify the URL just once. Use best effort. Timeout hard coded to 5
- * seconds.
+ * Notify the URL just once. Use best effort.
*/
protected boolean notifyURLOnce() {
boolean success = false;
@@ -121,8 +125,8 @@ public class JobEndNotifier implements C
Log.info("Job end notification trying " + urlToNotify);
HttpURLConnection conn =
(HttpURLConnection) urlToNotify.openConnection(proxyToUse);
- conn.setConnectTimeout(5*1000);
- conn.setReadTimeout(5*1000);
+ conn.setConnectTimeout(timeout);
+ conn.setReadTimeout(timeout);
conn.setAllowUserInteraction(false);
if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
Log.warn("Job end notification to " + urlToNotify +" failed with code: "
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java?rev=1470760&r1=1470759&r2=1470760&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java Tue Apr 23 00:14:06 2013
@@ -73,6 +73,13 @@ public class TestJobEndNotifier extends
+ waitInterval, waitInterval == 5000);
}
+ private void testTimeout(Configuration conf) {
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_TIMEOUT, "1000");
+ setConf(conf);
+ Assert.assertTrue("Expected timeout to be 1000, but was "
+ + timeout, timeout == 1000);
+ }
+
private void testProxyConfiguration(Configuration conf) {
conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "somehost");
setConf(conf);
@@ -109,6 +116,7 @@ public class TestJobEndNotifier extends
Configuration conf = new Configuration();
testNumRetries(conf);
testWaitInterval(conf);
+ testTimeout(conf);
testProxyConfiguration(conf);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java?rev=1470760&r1=1470759&r2=1470760&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java Tue Apr 23 00:14:06 2013
@@ -44,9 +44,10 @@ public class JobEndNotifier {
JobEndStatusInfo notification = null;
String uri = conf.getJobEndNotificationURI();
if (uri != null) {
- // +1 to make logic for first notification identical to a retry
- int retryAttempts = conf.getInt(JobContext.MR_JOB_END_RETRY_ATTEMPTS, 0) + 1;
+ int retryAttempts = conf.getInt(JobContext.MR_JOB_END_RETRY_ATTEMPTS, 0);
long retryInterval = conf.getInt(JobContext.MR_JOB_END_RETRY_INTERVAL, 30000);
+ int timeout = conf.getInt(JobContext.MR_JOB_END_NOTIFICATION_TIMEOUT,
+ JobContext.DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT);
if (uri.contains("$jobId")) {
uri = uri.replace("$jobId", status.getJobID().toString());
}
@@ -56,17 +57,22 @@ public class JobEndNotifier {
(status.getRunState() == JobStatus.FAILED) ? "FAILED" : "KILLED";
uri = uri.replace("$jobStatus", statusStr);
}
- notification = new JobEndStatusInfo(uri, retryAttempts, retryInterval);
+ notification = new JobEndStatusInfo(
+ uri, retryAttempts, retryInterval, timeout);
}
return notification;
}
- private static int httpNotification(String uri) throws IOException {
+ private static int httpNotification(String uri, int timeout)
+ throws IOException {
URI url = new URI(uri, false);
- HttpClient m_client = new HttpClient();
+ HttpClient httpClient = new HttpClient();
+ httpClient.getParams().setSoTimeout(timeout);
+ httpClient.getParams().setConnectionManagerTimeout(timeout);
+
HttpMethod method = new GetMethod(url.getEscapedURI());
method.setRequestHeader("Accept", "*/*");
- return m_client.executeMethod(method);
+ return httpClient.executeMethod(method);
}
// for use by the LocalJobRunner, without using a thread&queue,
@@ -74,9 +80,10 @@ public class JobEndNotifier {
public static void localRunnerNotification(JobConf conf, JobStatus status) {
JobEndStatusInfo notification = createNotification(conf, status);
if (notification != null) {
- while (notification.configureForRetry()) {
+ do {
try {
- int code = httpNotification(notification.getUri());
+ int code = httpNotification(notification.getUri(),
+ notification.getTimeout());
if (code != 200) {
throw new IOException("Invalid response status code: " + code);
}
@@ -96,7 +103,7 @@ public class JobEndNotifier {
catch (InterruptedException iex) {
LOG.error("Notification retry error [" + notification + "]", iex);
}
- }
+ } while (notification.configureForRetry());
}
}
@@ -105,12 +112,15 @@ public class JobEndNotifier {
private int retryAttempts;
private long retryInterval;
private long delayTime;
+ private int timeout;
- JobEndStatusInfo(String uri, int retryAttempts, long retryInterval) {
+ JobEndStatusInfo(String uri, int retryAttempts, long retryInterval,
+ int timeout) {
this.uri = uri;
this.retryAttempts = retryAttempts;
this.retryInterval = retryInterval;
this.delayTime = System.currentTimeMillis();
+ this.timeout = timeout;
}
public String getUri() {
@@ -125,6 +135,10 @@ public class JobEndNotifier {
return retryInterval;
}
+ public int getTimeout() {
+ return timeout;
+ }
+
public boolean configureForRetry() {
boolean retry = false;
if (getRetryAttempts() > 0) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1470760&r1=1470759&r2=1470760&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Tue Apr 23 00:14:06 2013
@@ -616,6 +616,9 @@ public interface MRJobConfig {
public static final String MR_JOB_END_NOTIFICATION_PROXY =
"mapreduce.job.end-notification.proxy";
+ public static final String MR_JOB_END_NOTIFICATION_TIMEOUT =
+ "mapreduce.job.end-notification.timeout";
+
public static final String MR_JOB_END_RETRY_ATTEMPTS =
"mapreduce.job.end-notification.retry.attempts";
@@ -628,6 +631,9 @@ public interface MRJobConfig {
public static final String MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL =
"mapreduce.job.end-notification.max.retry.interval";
+ public static final int DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT =
+ 5000;
+
/*
* MR AM Service Authorization
*/
Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1470195-1470759