You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2011/09/22 21:23:16 UTC
svn commit: r1174331 - in /hadoop/common/branches/branch-0.20-security:
CHANGES.txt src/mapred/org/apache/hadoop/mapred/TaskTracker.java
src/test/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java
Author: eli
Date: Thu Sep 22 19:23:15 2011
New Revision: 1174331
URL: http://svn.apache.org/viewvc?rev=1174331&view=rev
Log:
MAPREDUCE-2932. Missing instrumentation plugin class shouldn't crash the TT startup per design. Contributed by Harsh J Chouraria.
Added:
hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1174331&r1=1174330&r2=1174331&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Thu Sep 22 19:23:15 2011
@@ -21,6 +21,9 @@ Release 0.20.206.0 - unreleased
HADOOP-7653. tarball doesn't include .eclipse.templates.
(Jonathan Natkins via eli)
+ MAPREDUCE-2932. Missing instrumentation plugin class shouldn't
+ crash the TT startup per design. (harsh via eli)
+
IMPROVEMENTS
MAPREDUCE-2836. Provide option to fail jobs when submitted to
Modified: hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1174331&r1=1174330&r2=1174331&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Thu Sep 22 19:23:15 2011
@@ -864,21 +864,23 @@ public class TaskTracker implements MRCo
}
private void createInstrumentation() {
- Class<? extends TaskTrackerInstrumentation> metricsInst =
- getInstrumentationClass(fConf);
- LOG.debug("instrumentation class="+ metricsInst);
- if (metricsInst == null) {
- myInstrumentation = TaskTrackerInstrumentation.create(this);
- return;
- }
try {
+ Class<? extends TaskTrackerInstrumentation> metricsInst =
+ getInstrumentationClass(fConf);
+ LOG.debug("instrumentation class="+ metricsInst);
+ if (metricsInst == null) {
+ myInstrumentation = TaskTrackerInstrumentation.create(this);
+ return;
+ }
java.lang.reflect.Constructor<? extends TaskTrackerInstrumentation> c =
metricsInst.getConstructor(new Class<?>[] {TaskTracker.class} );
this.myInstrumentation = c.newInstance(this);
} catch(Exception e) {
//Reflection can throw lots of exceptions -- handle them all by
//falling back on the default.
- LOG.error("failed to initialize taskTracker metrics", e);
+ LOG.error(
+ "Failed to initialize taskTracker metrics, falling back to default.",
+ e);
this.myInstrumentation = TaskTrackerInstrumentation.create(this);
}
Added: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java?rev=1174331&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java (added)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java Thu Sep 22 19:23:15 2011
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.examples.SleepJob;
+import org.junit.Test;
+
+public class TestTaskTrackerInstrumentation {
+
+ @Test
+ public void testStartup() throws IOException {
+ MiniMRCluster mr = null;
+ try {
+ JobConf jtConf = new JobConf();
+ // Set a bad class.
+ jtConf.set("mapred.tasktracker.instrumentation",
+ "org.nowhere.FUBAR");
+ mr = new MiniMRCluster(1, "file:///", 1, null, null, jtConf);
+ // Assert that the TT fell back to default class.
+ TaskTracker tt = mr.getTaskTrackerRunner(0).getTaskTracker();
+ assertEquals(TaskTrackerInstrumentation.create(tt).getClass(),
+ mr.getTaskTrackerRunner(0).getTaskTracker()
+ .getTaskTrackerInstrumentation().getClass());
+ } finally {
+ mr.shutdown();
+ }
+ }
+
+ @Test
+ public void testSlots() throws IOException {
+ MiniMRCluster mr = null;
+ try {
+ JobConf jtConf = new JobConf();
+ jtConf.set("mapred.tasktracker.instrumentation",
+ MyTaskTrackerMetricsInst.class.getName());
+ mr = new MiniMRCluster(1, "file:///", 1, null, null, jtConf);
+ MyTaskTrackerMetricsInst instr = (MyTaskTrackerMetricsInst)
+ mr.getTaskTrackerRunner(0).getTaskTracker()
+ .getTaskTrackerInstrumentation();
+
+ JobConf conf = mr.createJobConf();
+ SleepJob job = new SleepJob();
+ job.setConf(conf);
+ int numMapTasks = 3;
+ int numReduceTasks = 2;
+ job.run(numMapTasks, numReduceTasks, 1, 1, 1, 1);
+
+ synchronized (instr) {
+ // 5 regular tasks + 2 setup/cleanup tasks.
+ assertEquals(7, instr.complete);
+ assertEquals(7, instr.end);
+ assertEquals(7, instr.launch);
+ }
+ } finally {
+ if (mr != null) {
+ mr.shutdown();
+ }
+ }
+ }
+
+ static class MyTaskTrackerMetricsInst extends TaskTrackerInstrumentation {
+ public int complete = 0;
+ public int launch = 0;
+ public int end = 0;
+
+ public MyTaskTrackerMetricsInst(TaskTracker tracker) {
+ super(tracker);
+ }
+
+ @Override
+ public void completeTask(TaskAttemptID t) {
+ this.complete++;
+ }
+
+ @Override
+ public void reportTaskLaunch(TaskAttemptID t, File stdout, File stderr) {
+ this.launch++;
+ }
+
+ @Override
+ public void reportTaskEnd(TaskAttemptID t) {
+ this.end++;
+ }
+ }
+}