You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2016/12/29 01:35:14 UTC
[1/3] hadoop git commit: Revert "MAPREDUCE-6286. A typo in
HistoryViewer makes some code useless,
which causes counter limits are not reset correctly. Contributed by Zhihai
Xu."
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 e25363300 -> 5f36b6226
Revert "MAPREDUCE-6286. A typo in HistoryViewer makes some code useless, which causes counter limits are not reset correctly. Contributed by Zhihai Xu."
This reverts commit 7c72c7f5290f0991f65102c2bd413f965a8ca409.
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8d78013
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8d78013
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8d78013
Branch: refs/heads/branch-2.8
Commit: e8d78013197735efc3d2dbede5a8f922169eb41a
Parents: e253633
Author: Junping Du <jd...@jduMBP.local>
Authored: Wed Dec 28 17:05:17 2016 -0800
Committer: Junping Du <jd...@jduMBP.local>
Committed: Wed Dec 28 17:05:17 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8d78013/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
index bbb08de..5a55aba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
@@ -93,7 +93,7 @@ public class HistoryViewer {
final Configuration jobConf = new Configuration(conf);
try {
jobConf.addResource(fs.open(jobConfPath), jobConfPath.toString());
- Limits.reset(jobConf);
+ Limits.reset(conf);
} catch (FileNotFoundException fnf) {
if (LOG.isWarnEnabled()) {
LOG.warn("Missing job conf in history", fnf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: Revert "MAPREDUCE-5875. Make Counter limits
consistent across JobClient, MRAppMaster,
and YarnChild. (Gera Shegalov via kasha)"
Posted by ju...@apache.org.
Revert "MAPREDUCE-5875. Make Counter limits consistent across JobClient, MRAppMaster, and YarnChild. (Gera Shegalov via kasha)"
This reverts commit 7bfd9e068d205f752c05d51778a4a4702329d31e.
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f36b622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f36b622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f36b622
Branch: refs/heads/branch-2.8
Commit: 5f36b62265ee4331f1c597f6031e623cd0e44c68
Parents: 8755a6f
Author: Junping Du <jd...@jduMBP.local>
Authored: Wed Dec 28 17:37:20 2016 -0800
Committer: Junping Du <jd...@jduMBP.local>
Committed: Wed Dec 28 17:37:20 2016 -0800
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 3 -
.../org/apache/hadoop/mapreduce/Cluster.java | 16 ++--
.../apache/hadoop/mapreduce/JobSubmitter.java | 2 -
.../hadoop/mapreduce/counters/Limits.java | 5 --
.../mapreduce/jobhistory/HistoryViewer.java | 13 ---
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 15 ----
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 87 +++-----------------
7 files changed, 21 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 0b7decc..779d061 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -60,7 +60,6 @@ import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent;
import org.apache.hadoop.mapreduce.jobhistory.EventReader;
import org.apache.hadoop.mapreduce.jobhistory.EventType;
@@ -1229,8 +1228,6 @@ public class MRAppMaster extends CompositeService {
// finally set the job classloader
MRApps.setClassLoader(jobClassLoader, getConfig());
- // set job classloader if configured
- Limits.init(getConfig());
if (initFailed) {
JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
index c4c4b6b..de3f3b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
@@ -206,15 +206,15 @@ public class Cluster {
public Job getJob(JobID jobId) throws IOException, InterruptedException {
JobStatus status = client.getJobStatus(jobId);
if (status != null) {
- final JobConf conf = new JobConf();
- final Path jobPath = new Path(client.getFilesystemName(),
- status.getJobFile());
- final FileSystem fs = FileSystem.get(jobPath.toUri(), getConf());
+ JobConf conf;
try {
- conf.addResource(fs.open(jobPath), jobPath.toString());
- } catch (FileNotFoundException fnf) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Job conf missing on cluster", fnf);
+ conf = new JobConf(status.getJobFile());
+ } catch (RuntimeException ex) {
+ // If job file doesn't exist it means we can't find the job
+ if (ex.getCause() instanceof FileNotFoundException) {
+ return null;
+ } else {
+ throw ex;
}
}
return Job.getInstance(this, status, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 18b76a1..ddd1cb4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.mapred.QueueACL;
import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
-import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.TokenCache;
@@ -235,7 +234,6 @@ class JobSubmitter {
// Write job file to submit dir
writeConf(conf, submitJobFile);
- Limits.reset(conf);
//
// Now, actually submit the job (using the submit name)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
index 3821694..34b0fae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
@@ -123,9 +123,4 @@ public class Limits {
public synchronized LimitExceededException violation() {
return firstViolation;
}
-
- public static synchronized void reset(Configuration conf) {
- isInited = false;
- init(conf);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
index 5a55aba..3b00b8c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.mapreduce.jobhistory;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.text.DecimalFormat;
import java.text.Format;
@@ -44,7 +43,6 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
import org.apache.hadoop.mapreduce.util.HostUtil;
@@ -88,17 +86,6 @@ public class HistoryViewer {
System.err.println("Ignore unrecognized file: " + jobFile.getName());
throw new IOException(errorMsg);
}
- final Path jobConfPath = new Path(jobFile.getParent(), jobDetails[0]
- + "_" + jobDetails[1] + "_" + jobDetails[2] + "_conf.xml");
- final Configuration jobConf = new Configuration(conf);
- try {
- jobConf.addResource(fs.open(jobConfPath), jobConfPath.toString());
- Limits.reset(conf);
- } catch (FileNotFoundException fnf) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Missing job conf in history", fnf);
- }
- }
JobHistoryParser parser = new JobHistoryParser(fs, jobFile);
job = parser.parse();
jobId = job.getJobId().toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
index f416402..36fcdf8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.mapreduce.v2.hs;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -35,7 +34,6 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.TaskCompletionEvent;
@@ -43,7 +41,6 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
@@ -352,19 +349,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
if (historyFileAbsolute != null) {
JobHistoryParser parser = null;
try {
- final FileSystem fs = historyFileAbsolute.getFileSystem(conf);
parser = createJobHistoryParser(historyFileAbsolute);
- final Path jobConfPath = new Path(historyFileAbsolute.getParent(),
- JobHistoryUtils.getIntermediateConfFileName(jobId));
- final Configuration conf = new Configuration();
- try {
- conf.addResource(fs.open(jobConfPath), jobConfPath.toString());
- Limits.reset(conf);
- } catch (FileNotFoundException fnf) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Missing job conf in history", fnf);
- }
- }
this.jobInfo = parser.parse();
} catch (IOException e) {
throw new YarnRuntimeException("Could not load history file "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f36b622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 9a4eb67..84013e2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -55,14 +55,10 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
@@ -114,7 +110,6 @@ public class TestMRJobs {
EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);
private static final int NUM_NODE_MGRS = 3;
private static final String TEST_IO_SORT_MB = "11";
- private static final String TEST_GROUP_MAX = "200";
private static final int DEFAULT_REDUCES = 2;
protected int numSleepReducers = DEFAULT_REDUCES;
@@ -307,58 +302,31 @@ public class TestMRJobs {
}
@Test(timeout = 300000)
- public void testConfVerificationWithClassloader() throws Exception {
- testConfVerification(true, false, false, false);
- }
-
- @Test(timeout = 300000)
- public void testConfVerificationWithClassloaderCustomClasses()
- throws Exception {
- testConfVerification(true, true, false, false);
- }
-
- @Test(timeout = 300000)
- public void testConfVerificationWithOutClassloader() throws Exception {
- testConfVerification(false, false, false, false);
- }
-
- @Test(timeout = 300000)
- public void testConfVerificationWithJobClient() throws Exception {
- testConfVerification(false, false, true, false);
+ public void testJobClassloader() throws IOException, InterruptedException,
+ ClassNotFoundException {
+ testJobClassloader(false);
}
@Test(timeout = 300000)
- public void testConfVerificationWithJobClientLocal() throws Exception {
- testConfVerification(false, false, true, true);
+ public void testJobClassloaderWithCustomClasses() throws IOException,
+ InterruptedException, ClassNotFoundException {
+ testJobClassloader(true);
}
- private void testConfVerification(boolean useJobClassLoader,
- boolean useCustomClasses, boolean useJobClientForMonitring,
- boolean useLocal) throws Exception {
- LOG.info("\n\n\nStarting testConfVerification()"
- + " jobClassloader=" + useJobClassLoader
- + " customClasses=" + useCustomClasses
- + " jobClient=" + useJobClientForMonitring
- + " localMode=" + useLocal);
+ private void testJobClassloader(boolean useCustomClasses) throws IOException,
+ InterruptedException, ClassNotFoundException {
+ LOG.info("\n\n\nStarting testJobClassloader()"
+ + " useCustomClasses=" + useCustomClasses);
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ " not found. Not running test.");
return;
}
- final Configuration clusterConfig;
- if (useLocal) {
- clusterConfig = new Configuration();
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
- } else {
- clusterConfig = mrCluster.getConfig();
- }
- final JobClient jc = new JobClient(clusterConfig);
- final Configuration sleepConf = new Configuration(clusterConfig);
+ final Configuration sleepConf = new Configuration(mrCluster.getConfig());
// set master address to local to test that local mode applied iff framework == local
sleepConf.set(MRConfig.MASTER_ADDRESS, "local");
- sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER,
- useJobClassLoader);
+ sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
if (useCustomClasses) {
// to test AM loading user classes such as output format class, we want
// to blacklist them from the system classes (they need to be prepended
@@ -376,7 +344,6 @@ public class TestMRJobs {
sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
sleepConf.set(MRJobConfig.REDUCE_LOG_LEVEL, Level.ALL.toString());
sleepConf.set(MRJobConfig.MAP_JAVA_OPTS, "-verbose:class");
- sleepConf.set(MRJobConfig.COUNTER_GROUPS_MAX_KEY, TEST_GROUP_MAX);
final SleepJob sleepJob = new SleepJob();
sleepJob.setConf(sleepConf);
final Job job = sleepJob.createJob(1, 1, 10, 1, 10, 1);
@@ -394,26 +361,7 @@ public class TestMRJobs {
jobConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, true);
}
job.submit();
- final boolean succeeded;
- if (useJobClientForMonitring && !useLocal) {
- // We can't use getJobID in useLocal case because JobClient and Job
- // point to different instances of LocalJobRunner
- //
- final JobID mapredJobID = JobID.downgrade(job.getJobID());
- RunningJob runningJob = null;
- do {
- Thread.sleep(10);
- runningJob = jc.getJob(mapredJobID);
- } while (runningJob == null);
- Assert.assertEquals("Unexpected RunningJob's "
- + MRJobConfig.COUNTER_GROUPS_MAX_KEY,
- TEST_GROUP_MAX, runningJob.getConfiguration()
- .get(MRJobConfig.COUNTER_GROUPS_MAX_KEY));
- runningJob.waitForCompletion();
- succeeded = runningJob.isSuccessful();
- } else {
- succeeded = job.waitForCompletion(true);
- }
+ boolean succeeded = job.waitForCompletion(true);
Assert.assertTrue("Job status: " + job.getStatus().getFailureInfo(),
succeeded);
}
@@ -1194,14 +1142,5 @@ public class TestMRJobs {
+ ", actual: " + ioSortMb);
}
}
-
- @Override
- public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException {
- super.map(key, value, context);
- for (int i = 0; i < 100; i++) {
- context.getCounter("testCounterGroup-" + i,
- "testCounter").increment(1);
- }
- }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: Revert "MAPREDUCE-6199. AbstractCounters are
not reset completely on deserialization (adhoot via rkanter)"
Posted by ju...@apache.org.
Revert "MAPREDUCE-6199. AbstractCounters are not reset completely on deserialization (adhoot via rkanter)"
This reverts commit f9341c1e2cffc4e3ac37cb7da4e535ca5c6d3913.
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8755a6f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8755a6f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8755a6f5
Branch: refs/heads/branch-2.8
Commit: 8755a6f5ac6785d6ed521555a2187389360bac42
Parents: e8d7801
Author: Junping Du <jd...@jduMBP.local>
Authored: Wed Dec 28 17:13:04 2016 -0800
Committer: Junping Du <jd...@jduMBP.local>
Committed: Wed Dec 28 17:13:04 2016 -0800
----------------------------------------------------------------------
.../mapreduce/counters/AbstractCounters.java | 4 --
.../hadoop/mapreduce/counters/Limits.java | 7 ----
.../apache/hadoop/mapreduce/TestCounters.java | 39 +-------------------
3 files changed, 1 insertion(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8755a6f5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
index e6e74da..4ab7e89 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
@@ -307,10 +307,6 @@ public abstract class AbstractCounters<C extends Counter,
fgroups.put(group.getName(), group);
}
int numGroups = WritableUtils.readVInt(in);
- if (!groups.isEmpty()) {
- groups.clear();
- limits.reset();
- }
while (numGroups-- > 0) {
limits.checkGroups(groups.size() + 1);
G group = groupFactory.newGenericGroup(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8755a6f5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
index 9546c8d..3821694 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
@@ -124,15 +124,8 @@ public class Limits {
return firstViolation;
}
- // This allows initialization of global settings and not for an instance
public static synchronized void reset(Configuration conf) {
isInited = false;
init(conf);
}
-
- // This allows resetting of an instance to allow reuse
- public synchronized void reset() {
- totalCounters = 0;
- firstViolation = null;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8755a6f5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
index 0215568..83d689c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
@@ -17,12 +17,8 @@
*/
package org.apache.hadoop.mapreduce;
-import java.io.IOException;
import java.util.Random;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -74,40 +70,7 @@ public class TestCounters {
testMaxGroups(new Counters());
}
}
-
- @Test public void testResetOnDeserialize() throws IOException {
- // Allow only one counterGroup
- Configuration conf = new Configuration();
- conf.setInt(MRJobConfig.COUNTER_GROUPS_MAX_KEY, 1);
- Limits.init(conf);
-
- Counters countersWithOneGroup = new Counters();
- countersWithOneGroup.findCounter("firstOf1Allowed", "First group");
- boolean caughtExpectedException = false;
- try {
- countersWithOneGroup.findCounter("secondIsTooMany", "Second group");
- }
- catch (LimitExceededException _) {
- caughtExpectedException = true;
- }
-
- assertTrue("Did not throw expected exception",
- caughtExpectedException);
-
- Counters countersWithZeroGroups = new Counters();
- DataOutputBuffer out = new DataOutputBuffer();
- countersWithZeroGroups.write(out);
-
- DataInputBuffer in = new DataInputBuffer();
- in.reset(out.getData(), out.getLength());
-
- countersWithOneGroup.readFields(in);
-
- // After reset one should be able to add a group
- countersWithOneGroup.findCounter("firstGroupAfterReset", "After reset " +
- "limit should be set back to zero");
- }
-
+
@Test
public void testCountersIncrement() {
Counters fCounters = new Counters();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org