You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2014/02/28 20:37:49 UTC
svn commit: r1573043 - in
/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/tes...
Author: szetszwo
Date: Fri Feb 28 19:37:47 2014
New Revision: 1573043
URL: http://svn.apache.org/r1573043
Log:
Merge r1569890 through r1573042 from trunk.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1572723-1573042
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt?rev=1573043&r1=1573042&r2=1573043&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt Fri Feb 28 19:37:47 2014
@@ -153,6 +153,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5688. TestStagingCleanup fails intermittently with JDK7 (Mit
Desai via jeagles)
+ MAPREDUCE-5665. Add audience annotations to MiniMRYarnCluster and
+ MiniMRCluster. (Anubhav Dhoot via kasha)
+
OPTIMIZATIONS
BUG FIXES
@@ -199,6 +202,9 @@ Release 2.4.0 - UNRELEASED
to the YARN's web-app proxy with the correct scheme prefix. (Jian He via
vinodkv)
+ MAPREDUCE-5768. TestMRJobs.testContainerRollingLog fails on trunk (Gera
+ Shegalov via jlowe)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1572723-1573042
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java?rev=1573043&r1=1573042&r2=1573043&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java Fri Feb 28 19:37:47 2014
@@ -22,6 +22,8 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -40,6 +42,8 @@ import org.apache.hadoop.security.UserGr
* instead
*/
@Deprecated
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class MiniMRCluster {
private static final Log LOG = LogFactory.getLog(MiniMRCluster.class);
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1573043&r1=1573042&r2=1573043&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Fri Feb 28 19:37:47 2014
@@ -26,6 +26,8 @@ import java.net.UnknownHostException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileContext;
@@ -55,6 +57,8 @@ import org.apache.hadoop.yarn.webapp.uti
* Configures and starts the MR-specific components in the YARN cluster.
*
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class MiniMRYarnCluster extends MiniYARNCluster {
public static final String APPJAR = JarFinder.getJar(LocalContainerLauncher.class);
Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1573043&r1=1573042&r2=1573043&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Fri Feb 28 19:37:47 2014
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.InputStreamReader;
import java.io.IOException;
import java.io.StringReader;
import java.net.URI;
@@ -82,8 +81,10 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -492,28 +493,17 @@ public class TestMRJobs {
LOG.info("Checking for glob: " + absSyslogGlob);
final FileStatus[] syslogs = localFs.globStatus(absSyslogGlob);
for (FileStatus slog : syslogs) {
- // check all syslogs for the container
- //
- final FileStatus[] sysSiblings = localFs.globStatus(new Path(
- slog.getPath().getParent(), TaskLog.LogName.SYSLOG + "*"));
- boolean foundAppMaster = false;
- floop:
- for (FileStatus f : sysSiblings) {
- final BufferedReader reader = new BufferedReader(
- new InputStreamReader(localFs.open(f.getPath())));
- String line;
- try {
- while ((line = reader.readLine()) != null) {
- if (line.contains(MRJobConfig.APPLICATION_MASTER_CLASS)) {
- foundAppMaster = true;
- break floop;
- }
- }
- } finally {
- reader.close();
- }
+ boolean foundAppMaster = job.isUber();
+ final Path containerPathComponent = slog.getPath().getParent();
+ if (!foundAppMaster) {
+ final ContainerId cid = ConverterUtils.toContainerId(
+ containerPathComponent.getName());
+ foundAppMaster = (cid.getId() == 1);
}
+ final FileStatus[] sysSiblings = localFs.globStatus(new Path(
+ containerPathComponent, TaskLog.LogName.SYSLOG + "*"));
+
if (foundAppMaster) {
numAppMasters++;
} else {