You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/07/30 03:26:28 UTC

[1/2] tez git commit: TEZ-2441. Add tests for TezTaskRunner2. (sseth)

Repository: tez
Updated Branches:
  refs/heads/TEZ-2003 aed5a6804 -> a169d8ae9


TEZ-2441. Add tests for TezTaskRunner2. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/509238b7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/509238b7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/509238b7

Branch: refs/heads/TEZ-2003
Commit: 509238b7239052a86d6880110782424f93869e0f
Parents: aed5a68
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Jul 29 18:25:18 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Jul 29 18:25:18 2015 -0700

----------------------------------------------------------------------
 TEZ-2003-CHANGES.txt                            |   1 +
 .../org/apache/tez/runtime/task/TezChild.java   |   5 +-
 .../apache/tez/runtime/task/TezTaskRunner.java  |   2 +-
 .../apache/tez/runtime/task/TezTaskRunner2.java |  42 +-
 .../runtime/task/TaskExecutionTestHelpers.java  | 451 +++++++++++++
 .../runtime/task/TestContainerExecution.java    |  59 ++
 .../tez/runtime/task/TestTaskExecution.java     | 400 +-----------
 .../tez/runtime/task/TestTaskExecution2.java    | 638 +++++++++++++++++++
 .../src/test/resources/log4j.properties         |  19 +
 9 files changed, 1213 insertions(+), 404 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index b88044b..9d72d92 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -39,5 +39,6 @@ ALL CHANGES:
   TEZ-2651. Pluggable services should not extend AbstractService.
   TEZ-2652. Cleanup the way services are specified for an AM and vertices.
   TEZ-2653. Change service contexts to expose a user specified payload instead of the AM configuration.
+  TEZ-2441. Add tests for TezTaskRunner2.
 
 INCOMPATIBLE CHANGES:

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 353fe23..b64ec37 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.log4j.LogManager;
 import org.apache.tez.common.ContainerContext;
 import org.apache.tez.common.ContainerTask;
 import org.apache.tez.common.TezCommonUtils;
@@ -68,7 +67,6 @@ import org.apache.tez.dag.utils.RelocalizationUtils;
 import org.apache.tez.runtime.api.ExecutionContext;
 import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
 import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezUmbilical;
 import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl;
 import org.apache.tez.runtime.internals.api.TaskReporterInterface;
 import org.slf4j.Logger;
@@ -256,6 +254,7 @@ public class TezChild {
         boolean shouldDie;
         try {
           TaskRunner2Result result = taskRunner.run();
+          LOG.info("TaskRunner2Result: {}", result);
           shouldDie = result.isContainerShutdownRequested();
           if (shouldDie) {
             LOG.info("Got a shouldDie notification via heartbeats for container {}. Shutting down", containerIdString);
@@ -377,8 +376,6 @@ public class TezChild {
       }
       if (ownUmbilical) {
         RPC.stopProxy(umbilical);
-        // TODO Temporary change. Revert. Ideally, move this over to the main method in TezChild if possible.
-//        LogManager.shutdown();
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
index a82d87b..aebf6a9 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java
@@ -250,7 +250,7 @@ public class TezTaskRunner implements TezUmbilical, ErrorReporter {
                   cause = ((UndeclaredThrowableException) cause).getCause();
                 }
                 maybeRegisterFirstException(cause);
-                LOG.info("Encounted an error while executing task: " + task.getTaskAttemptID(),
+                LOG.info("Encountered an error while executing task: " + task.getTaskAttemptID(),
                     cause);
                 try {
                   sendFailure(cause, "Failure while running task");

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java
index a5fabb5..1a8828d 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java
@@ -24,6 +24,7 @@ import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Multimap;
 import com.google.common.util.concurrent.ListenableFuture;
@@ -48,6 +49,9 @@ import org.slf4j.LoggerFactory;
 
 public class TezTaskRunner2 {
 
+  // Behaviour changes as compared to TezTaskRunner
+  // - Exception not thrown. Instead returned in the result.
+  // - The actual exception is part of the result, instead of requiring a getCause().
 
   private static final Logger LOG = LoggerFactory.getLogger(TezTaskRunner2.class);
 
@@ -156,19 +160,7 @@ public class TezTaskRunner2 {
           }
         }
       }
-      if (executionResult != null) {
-        synchronized (this) {
-          if (isRunningState()) {
-            if (executionResult.error != null) {
-              trySettingEndReason(EndReason.TASK_ERROR);
-              registerFirstException(executionResult.error, null);
-            } else {
-              trySettingEndReason(EndReason.SUCCESS);
-              taskComplete.set(true);
-            }
-          }
-        }
-      }
+      processCallableResult(executionResult);
 
       switch (firstEndReason) {
         case SUCCESS:
@@ -249,6 +241,26 @@ public class TezTaskRunner2 {
     }
   }
 
+  // It's possible for the task to actually complete, and an alternate signal such as killTask/killContainer
+  // come in before the future has been processed by this thread. That condition is not handled - and
+  // the result of the execution will be determind by the thread order.
+  @VisibleForTesting
+  void processCallableResult(TaskRunner2CallableResult executionResult) {
+    if (executionResult != null) {
+      synchronized (this) {
+        if (isRunningState()) {
+          if (executionResult.error != null) {
+            trySettingEndReason(EndReason.TASK_ERROR);
+            registerFirstException(executionResult.error, null);
+          } else {
+            trySettingEndReason(EndReason.SUCCESS);
+            taskComplete.set(true);
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Attempt to kill the running task, if it hasn't already completed for some other reason.
    * @return true if the task kill was honored, false otherwise
@@ -438,12 +450,12 @@ public class TezTaskRunner2 {
   private String getTaskDiagnosticsString(Throwable t, String message) {
     String diagnostics;
     if (t != null && message != null) {
-      diagnostics = "exceptionThrown=" + ExceptionUtils.getStackTrace(t) + ", errorMessage="
+      diagnostics = "Failure while running task: " + ExceptionUtils.getStackTrace(t) + ", errorMessage="
           + message;
     } else if (t == null && message == null) {
       diagnostics = "Unknown error";
     } else {
-      diagnostics = t != null ? "exceptionThrown=" + ExceptionUtils.getStackTrace(t)
+      diagnostics = t != null ? "Failure while running task: " + ExceptionUtils.getStackTrace(t)
           : " errorMessage=" + message;
     }
     return diagnostics;

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java
new file mode 100644
index 0000000..fc42da3
--- /dev/null
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.task;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.ContainerContext;
+import org.apache.tez.common.ContainerTask;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.api.LogicalOutput;
+import org.apache.tez.runtime.api.ProcessorContext;
+import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
+import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent;
+import org.apache.tez.runtime.api.impl.TezEvent;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TaskExecutionTestHelpers {
+
+  public static final String HEARTBEAT_EXCEPTION_STRING = "HeartbeatException";
+
+  // Uses static fields for signaling. Ensure only used by one test at a time.
+  public static class TestProcessor extends AbstractLogicalIOProcessor {
+
+    public static final byte[] CONF_EMPTY = new byte[] { 0 };
+    public static final byte[] CONF_THROW_IO_EXCEPTION = new byte[] { 1 };
+    public static final byte[] CONF_THROW_TEZ_EXCEPTION = new byte[] { 2 };
+    public static final byte[] CONF_SIGNAL_FATAL_AND_THROW = new byte[] { 4 };
+    public static final byte[] CONF_SIGNAL_FATAL_AND_LOOP = new byte[] { 8 };
+    public static final byte[] CONF_SIGNAL_FATAL_AND_COMPLETE = new byte[] { 16 };
+
+    private static final Logger LOG = LoggerFactory.getLogger(TestProcessor.class);
+
+    private static final ReentrantLock processorLock = new ReentrantLock();
+    private static final Condition processorCondition = processorLock.newCondition();
+    private static final Condition loopCondition = processorLock.newCondition();
+    private static final Condition completionCondition = processorLock.newCondition();
+    private static final Condition runningCondition = processorLock.newCondition();
+    private static volatile boolean completed = false;
+    private static volatile boolean running = false;
+    private static volatile boolean looping = false;
+    private static volatile boolean signalled = false;
+
+    private static boolean receivedInterrupt = false;
+    private static volatile boolean wasAborted = false;
+
+    private boolean throwIOException = false;
+    private boolean throwTezException = false;
+    private boolean signalFatalAndThrow = false;
+    private boolean signalFatalAndLoop = false;
+    private boolean signalFatalAndComplete = false;
+
+    public TestProcessor(ProcessorContext context) {
+      super(context);
+    }
+
+    @Override
+    public void initialize() throws Exception {
+      parseConf(getContext().getUserPayload().deepCopyAsArray());
+    }
+
+    @Override
+    public void handleEvents(List<Event> processorEvents) {
+
+    }
+
+    @Override
+    public void close() throws Exception {
+
+    }
+
+    private void parseConf(byte[] bytes) {
+      byte b = bytes[0];
+      throwIOException = (b & 1) > 0;
+      throwTezException = (b & 2) > 0;
+      signalFatalAndThrow = (b & 4) > 0;
+      signalFatalAndLoop = (b & 8) > 0;
+      signalFatalAndComplete = (b & 16) > 0;
+    }
+
+    public static void reset() {
+      signalled = false;
+      receivedInterrupt = false;
+      completed = false;
+      running = false;
+      wasAborted = false;
+    }
+
+    public static void signal() {
+      LOG.info("Signalled");
+      processorLock.lock();
+      try {
+        signalled = true;
+        processorCondition.signal();
+      } finally {
+        processorLock.unlock();
+      }
+    }
+
+    public static void awaitStart() throws InterruptedException {
+      LOG.info("Awaiting Process run");
+      processorLock.lock();
+      try {
+        if (running) {
+          return;
+        }
+        runningCondition.await();
+      } finally {
+        processorLock.unlock();
+      }
+    }
+
+    public static void awaitLoop() throws InterruptedException {
+      LOG.info("Awaiting loop after signalling error");
+      processorLock.lock();
+      try {
+        if (looping) {
+          return;
+        }
+        loopCondition.await();
+      } finally {
+        processorLock.unlock();
+      }
+    }
+
+    public static void awaitCompletion() throws InterruptedException {
+      LOG.info("Await completion");
+      processorLock.lock();
+      try {
+        if (completed) {
+          return;
+        } else {
+          completionCondition.await();
+        }
+      } finally {
+        processorLock.unlock();
+      }
+    }
+
+    public static boolean wasInterrupted() {
+      processorLock.lock();
+      try {
+        return receivedInterrupt;
+      } finally {
+        processorLock.unlock();
+      }
+    }
+
+    public static boolean wasAborted() {
+      processorLock.lock();
+      try {
+        return wasAborted;
+      } finally {
+        processorLock.unlock();
+      }
+    }
+
+    @Override
+    public void abort() {
+      wasAborted = true;
+    }
+
+    @Override
+    public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws
+        Exception {
+      processorLock.lock();
+      running = true;
+      runningCondition.signal();
+      try {
+        try {
+          LOG.info("Signal is: " + signalled);
+          if (!signalled) {
+            LOG.info("Waiting for processor signal");
+            processorCondition.await();
+          }
+          if (Thread.currentThread().isInterrupted()) {
+            throw new InterruptedException();
+          }
+          LOG.info("Received processor signal");
+          if (throwIOException) {
+            throw createProcessorIOException();
+          } else if (throwTezException) {
+            throw createProcessorTezException();
+          } else if (signalFatalAndThrow) {
+            IOException io = new IOException("FATALERROR");
+            getContext().fatalError(io, "FATALERROR");
+            throw io;
+          } else if (signalFatalAndComplete) {
+            IOException io = new IOException("FATALERROR");
+            getContext().fatalError(io, "FATALERROR");
+            return;
+          } else if (signalFatalAndLoop) {
+            IOException io = createProcessorIOException();
+            getContext().fatalError(io, "FATALERROR");
+            LOG.info("looping");
+            looping = true;
+            loopCondition.signal();
+            LOG.info("Waiting for Processor signal again");
+            processorCondition.await();
+            LOG.info("Received second processor signal");
+          }
+        } catch (InterruptedException e) {
+          receivedInterrupt = true;
+        }
+      } finally {
+        completed = true;
+        completionCondition.signal();
+        processorLock.unlock();
+      }
+    }
+  }
+
+  public static TezException createProcessorTezException() {
+    return new TezException("TezException");
+  }
+
+  public static IOException createProcessorIOException() {
+    return new IOException("IOException");
+  }
+
+  public static class TezTaskUmbilicalForTest implements TezTaskUmbilicalProtocol {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TezTaskUmbilicalForTest.class);
+
+    private final List<TezEvent> requestEvents = new LinkedList<TezEvent>();
+
+    private final ReentrantLock umbilicalLock = new ReentrantLock();
+    private final Condition eventCondition = umbilicalLock.newCondition();
+    private boolean pendingEvent = false;
+    private boolean eventEnacted = false;
+
+    volatile int getTaskInvocations = 0;
+
+    private boolean shouldThrowException = false;
+    private boolean shouldSendDieSignal = false;
+
+    public void signalThrowException() {
+      umbilicalLock.lock();
+      try {
+        shouldThrowException = true;
+        pendingEvent = true;
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void signalSendShouldDie() {
+      umbilicalLock.lock();
+      try {
+        shouldSendDieSignal = true;
+        pendingEvent = true;
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void awaitRegisteredEvent() throws InterruptedException {
+      umbilicalLock.lock();
+      try {
+        if (eventEnacted) {
+          return;
+        }
+        LOG.info("Awaiting event");
+        eventCondition.await();
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void resetTrackedEvents() {
+      umbilicalLock.lock();
+      try {
+        requestEvents.clear();
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void verifyNoCompletionEvents() {
+      umbilicalLock.lock();
+      try {
+        for (TezEvent event : requestEvents) {
+          if (event.getEvent() instanceof TaskAttemptFailedEvent) {
+            fail("Found a TaskAttemptFailedEvent when not expected");
+          }
+          if (event.getEvent() instanceof TaskAttemptCompletedEvent) {
+            fail("Found a TaskAttemptCompletedvent when not expected");
+          }
+        }
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void verifyTaskFailedEvent(String diagnostics) {
+      umbilicalLock.lock();
+      try {
+        for (TezEvent event : requestEvents) {
+          if (event.getEvent() instanceof TaskAttemptFailedEvent) {
+            TaskAttemptFailedEvent failedEvent = (TaskAttemptFailedEvent) event.getEvent();
+            if (failedEvent.getDiagnostics().startsWith(diagnostics)) {
+              return;
+            } else {
+              fail("Diagnostic message does not match expected message. Found [" +
+                  failedEvent.getDiagnostics() + "], Expected: [" + diagnostics + "]");
+            }
+          }
+        }
+        fail("No TaskAttemptFailedEvents sent over umbilical");
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void verifyTaskFailedEvent(String diagStart, String diagContains) {
+      umbilicalLock.lock();
+      try {
+        for (TezEvent event : requestEvents) {
+          if (event.getEvent() instanceof TaskAttemptFailedEvent) {
+            TaskAttemptFailedEvent failedEvent = (TaskAttemptFailedEvent) event.getEvent();
+            if (failedEvent.getDiagnostics().startsWith(diagStart)) {
+              if (diagContains != null) {
+                if (failedEvent.getDiagnostics().contains(diagContains)) {
+                  return;
+                } else {
+                  fail("Diagnostic message does not contain expected message. Found [" +
+                      failedEvent.getDiagnostics() + "], Expected: [" + diagContains + "]");
+                }
+              }
+            } else {
+              fail("Diagnostic message does not start with expected message. Found [" +
+                  failedEvent.getDiagnostics() + "], Expected: [" + diagStart + "]");
+            }
+          }
+        }
+        fail("No TaskAttemptFailedEvents sent over umbilical");
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    public void verifyTaskSuccessEvent() {
+      umbilicalLock.lock();
+      try {
+        for (TezEvent event : requestEvents) {
+          if (event.getEvent() instanceof TaskAttemptCompletedEvent) {
+            return;
+          }
+        }
+        fail("No TaskAttemptFailedEvents sent over umbilical");
+      } finally {
+        umbilicalLock.unlock();
+      }
+    }
+
+    @Override
+    public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+      return 0;
+    }
+
+    @Override
+    public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+                                                  int clientMethodsHash) throws IOException {
+      return null;
+    }
+
+    @Override
+    public ContainerTask getTask(ContainerContext containerContext) throws IOException {
+      // Return shouldDie = true
+      getTaskInvocations++;
+      return new ContainerTask(null, true, null, null, false);
+    }
+
+    @Override
+    public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
+      return true;
+    }
+
+    @Override
+    public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
+        TezException {
+      umbilicalLock.lock();
+      if (request.getEvents() != null) {
+        requestEvents.addAll(request.getEvents());
+      }
+      try {
+        if (shouldThrowException) {
+          LOG.info("TestUmbilical throwing Exception");
+          throw new IOException(HEARTBEAT_EXCEPTION_STRING);
+        }
+        TezHeartbeatResponse response = new TezHeartbeatResponse();
+        response.setLastRequestId(request.getRequestId());
+        if (shouldSendDieSignal) {
+          LOG.info("TestUmbilical returning shouldDie=true");
+          response.setShouldDie();
+        }
+        return response;
+      } finally {
+        if (pendingEvent) {
+          eventEnacted = true;
+          LOG.info("Signalling Event");
+          eventCondition.signal();
+        }
+        umbilicalLock.unlock();
+      }
+    }
+  }
+
+  public static ContainerId createContainerId(ApplicationId appId) {
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+    return containerId;
+  }
+
+  public static TaskReporter createTaskReporter(ApplicationId appId, TezTaskUmbilicalForTest umbilical) {
+    TaskReporter taskReporter = new TaskReporter(umbilical, 100, 1000, 100, new AtomicLong(0),
+        createContainerId(appId).toString());
+    return taskReporter;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java
new file mode 100644
index 0000000..c1616af
--- /dev/null
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.task;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.common.ContainerContext;
+import org.apache.tez.common.ContainerTask;
+import org.junit.Test;
+
+public class TestContainerExecution {
+
+  @Test(timeout = 5000)
+  public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+      ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      ContainerContext containerContext = new ContainerContext(containerId.toString());
+
+      ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
+      ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
+
+      getTaskFuture.get();
+      assertEquals(1, umbilical.getTaskInvocations);
+
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
index 1bcb337..a99416a 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
@@ -18,8 +18,8 @@
 
 package org.apache.tez.runtime.task;
 
+import static org.apache.tez.runtime.task.TaskExecutionTestHelpers.createTaskReporter;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -28,30 +28,18 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.common.ContainerContext;
-import org.apache.tez.common.ContainerTask;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
 import org.apache.tez.dag.api.ProcessorDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
@@ -60,21 +48,13 @@ import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
-import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.api.events.TaskAttemptCompletedEvent;
-import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent;
 import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
 import org.apache.tez.runtime.api.impl.InputSpec;
 import org.apache.tez.runtime.api.impl.OutputSpec;
 import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezEvent;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
 import org.apache.tez.runtime.common.resources.ScalingAllocator;
+import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TestProcessor;
+import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TezTaskUmbilicalForTest;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
@@ -82,7 +62,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.HashMultimap;
-import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 
@@ -91,7 +70,7 @@ public class TestTaskExecution {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestTaskExecution.class);
 
-  private static final String HEARTBEAT_EXCEPTION_STRING = "HeartbeatException";
+
 
   private static final Configuration defaultConf = new Configuration();
   private static final FileSystem localFs;
@@ -137,7 +116,7 @@ public class TestTaskExecution {
       TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           TestProcessor.CONF_EMPTY);
       // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       // Signal the processor to go through
       TestProcessor.signal();
       boolean result = taskRunnerFuture.get();
@@ -164,7 +143,7 @@ public class TestTaskExecution {
       TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           TestProcessor.CONF_EMPTY);
       // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       // Signal the processor to go through
       TestProcessor.signal();
       boolean result = taskRunnerFuture.get();
@@ -176,7 +155,7 @@ public class TestTaskExecution {
       taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           TestProcessor.CONF_EMPTY);
       // Setup the executor
-      taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       // Signal the processor to go through
       TestProcessor.signal();
       result = taskRunnerFuture.get();
@@ -188,7 +167,7 @@ public class TestTaskExecution {
     }
   }
 
-  // test tasked failed due to exception in Processor
+  // test task failed due to exception in Processor
   @Test(timeout = 5000)
   public void testFailedTask() throws IOException, InterruptedException, TezException {
 
@@ -203,7 +182,7 @@ public class TestTaskExecution {
       TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           TestProcessor.CONF_THROW_TEZ_EXCEPTION);
       // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       // Signal the processor to go through
       TestProcessor.awaitStart();
       TestProcessor.signal();
@@ -238,7 +217,7 @@ public class TestTaskExecution {
       TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           "NotExitedProcessor", TestProcessor.CONF_THROW_TEZ_EXCEPTION);
       // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       try {
         taskRunnerFuture.get();
       } catch (ExecutionException e) {
@@ -268,7 +247,7 @@ public class TestTaskExecution {
       TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           TestProcessor.CONF_EMPTY);
       // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       // Signal the processor to go through
       TestProcessor.awaitStart();
       umbilical.signalThrowException();
@@ -280,7 +259,7 @@ public class TestTaskExecution {
       } catch (ExecutionException e) {
         Throwable cause = e.getCause();
         assertTrue(cause instanceof IOException);
-        assertTrue(cause.getMessage().contains(HEARTBEAT_EXCEPTION_STRING));
+        assertTrue(cause.getMessage().contains(TaskExecutionTestHelpers.HEARTBEAT_EXCEPTION_STRING));
       }
       TestProcessor.awaitCompletion();
       assertTrue(TestProcessor.wasInterrupted());
@@ -307,7 +286,7 @@ public class TestTaskExecution {
       TezTaskRunner taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
           TestProcessor.CONF_EMPTY);
       // Setup the executor
-      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable(taskRunner));
+      Future<Boolean> taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable1ForTest(taskRunner));
       // Signal the processor to go through
       TestProcessor.awaitStart();
       umbilical.signalSendShouldDie();
@@ -329,38 +308,14 @@ public class TestTaskExecution {
     }
   }
 
-  @Test(timeout = 5000)
-  public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
-    ListeningExecutorService executor = null;
-    try {
-      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
-      executor = MoreExecutors.listeningDecorator(rawExecutor);
-      ApplicationId appId = ApplicationId.newInstance(10000, 1);
-      ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
-      ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
-
-      TezTaskUmbilicalForTest umbilical = new TezTaskUmbilicalForTest();
-      ContainerContext containerContext = new ContainerContext(containerId.toString());
-
-      ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
-      ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
-
-      getTaskFuture.get();
-      assertEquals(1, umbilical.getTaskInvocations);
-
-    } finally {
-      executor.shutdownNow();
-    }
-  }
-
   // Potential new tests
   // Different states - initialization failure, close failure
   // getTask states
 
-  private static class TaskRunnerCallable implements Callable<Boolean> {
+  private static class TaskRunnerCallable1ForTest implements Callable<Boolean> {
     private final TezTaskRunner taskRunner;
 
-    public TaskRunnerCallable(TezTaskRunner taskRunner) {
+    public TaskRunnerCallable1ForTest(TezTaskRunner taskRunner) {
       this.taskRunner = taskRunner;
     }
 
@@ -370,328 +325,9 @@ public class TestTaskExecution {
     }
   }
 
-  // Uses static fields for signaling. Ensure only used by one test at a time.
-  public static class TestProcessor extends AbstractLogicalIOProcessor {
-
-    public static final byte[] CONF_EMPTY = new byte[] { 0 };
-    public static final byte[] CONF_THROW_IO_EXCEPTION = new byte[] { 1 };
-    public static final byte[] CONF_THROW_TEZ_EXCEPTION = new byte[] { 2 };
-    public static final byte[] CONF_SIGNAL_FATAL_AND_THROW = new byte[] { 4 };
-    public static final byte[] CONF_SIGNAL_FATAL_AND_LOOP = new byte[] { 8 };
-    public static final byte[] CONF_SIGNAL_FATAL_AND_COMPLETE = new byte[] { 16 };
-
-    private static final Logger LOG = LoggerFactory.getLogger(TestProcessor.class);
-
-    private static final ReentrantLock processorLock = new ReentrantLock();
-    private static final Condition processorCondition = processorLock.newCondition();
-    private static final Condition completionCondition = processorLock.newCondition();
-    private static final Condition runningCondition = processorLock.newCondition();
-    private static boolean completed = false;
-    private static boolean running = false;
-    private static boolean signalled = false;
-
-    public static boolean receivedInterrupt = false;
-
-    private boolean throwIOException = false;
-    private boolean throwTezException = false;
-    private boolean signalFatalAndThrow = false;
-    private boolean signalFatalAndLoop = false;
-    private boolean signalFatalAndComplete = false;
-
-    public TestProcessor(ProcessorContext context) {
-      super(context);
-    }
-
-    @Override
-    public void initialize() throws Exception {
-      parseConf(getContext().getUserPayload().deepCopyAsArray());
-    }
-
-    @Override
-    public void handleEvents(List<Event> processorEvents) {
-
-    }
-
-    @Override
-    public void close() throws Exception {
-
-    }
 
-    private void parseConf(byte[] bytes) {
-      byte b = bytes[0];
-      throwIOException = (b & 1) > 1;
-      throwTezException = (b & 2) > 1;
-      signalFatalAndThrow = (b & 4) > 1;
-      signalFatalAndLoop = (b & 8) > 1;
-      signalFatalAndComplete = (b & 16) > 1;
-    }
 
-    public static void reset() {
-      signalled = false;
-      receivedInterrupt = false;
-      completed = false;
-      running = false;
-    }
 
-    public static void signal() {
-      LOG.info("Signalled");
-      processorLock.lock();
-      try {
-        signalled = true;
-        processorCondition.signal();
-      } finally {
-        processorLock.unlock();
-      }
-    }
-
-    public static void awaitStart() throws InterruptedException {
-      LOG.info("Awaiting Process run");
-      processorLock.lock();
-      try {
-        if (running) {
-          return;
-        }
-        runningCondition.await();
-      } finally {
-        processorLock.unlock();
-      }
-    }
-
-    public static void awaitCompletion() throws InterruptedException {
-      LOG.info("Await completion");
-      processorLock.lock();
-      try {
-        if (completed) {
-          return;
-        } else {
-          completionCondition.await();
-        }
-      } finally {
-        processorLock.unlock();
-      }
-    }
-
-    public static boolean wasInterrupted() {
-      processorLock.lock();
-      try {
-        return receivedInterrupt;
-      } finally {
-        processorLock.unlock();
-      }
-    }
-
-    @Override
-    public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws
-        Exception {
-      processorLock.lock();
-      running = true;
-      runningCondition.signal();
-      try {
-        try {
-          LOG.info("Signal is: " + signalled);
-          if (!signalled) {
-            LOG.info("Waiting for processor signal");
-            processorCondition.await();
-          }
-          if (Thread.currentThread().isInterrupted()) {
-            throw new InterruptedException();
-          }
-          LOG.info("Received processor signal");
-          if (throwIOException) {
-            throw new IOException();
-          } else if (throwTezException) {
-            throw new TezException("TezException");
-          } else if (signalFatalAndThrow) {
-            IOException io = new IOException("FATALERROR");
-            getContext().fatalError(io, "FATALERROR");
-            throw io;
-          } else if (signalFatalAndComplete) {
-            IOException io = new IOException("FATALERROR");
-            getContext().fatalError(io, "FATALERROR");
-            return;
-          } else if (signalFatalAndLoop) {
-            IOException io = new IOException("FATALERROR");
-            getContext().fatalError(io, "FATALERROR");
-            LOG.info("Waiting for Processor signal again");
-            processorCondition.await();
-            LOG.info("Received second processor signal");
-          }
-        } catch (InterruptedException e) {
-          receivedInterrupt = true;
-        }
-      } finally {
-        completed = true;
-        completionCondition.signal();
-        processorLock.unlock();
-      }
-    }
-  }
-
-  private static class TezTaskUmbilicalForTest implements TezTaskUmbilicalProtocol {
-
-    private static final Logger LOG = LoggerFactory.getLogger(TezTaskUmbilicalForTest.class);
-
-    private final List<TezEvent> requestEvents = new LinkedList<TezEvent>();
-
-    private final ReentrantLock umbilicalLock = new ReentrantLock();
-    private final Condition eventCondition = umbilicalLock.newCondition();
-    private boolean pendingEvent = false;
-    private boolean eventEnacted = false;
-
-    volatile int getTaskInvocations = 0;
-
-    private boolean shouldThrowException = false;
-    private boolean shouldSendDieSignal = false;
-
-    public void signalThrowException() {
-      umbilicalLock.lock();
-      try {
-        shouldThrowException = true;
-        pendingEvent = true;
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    public void signalSendShouldDie() {
-      umbilicalLock.lock();
-      try {
-        shouldSendDieSignal = true;
-        pendingEvent = true;
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    public void awaitRegisteredEvent() throws InterruptedException {
-      umbilicalLock.lock();
-      try {
-        if (eventEnacted) {
-          return;
-        }
-        LOG.info("Awaiting event");
-        eventCondition.await();
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    public void resetTrackedEvents() {
-      umbilicalLock.lock();
-      try {
-        requestEvents.clear();
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    public void verifyNoCompletionEvents() {
-      umbilicalLock.lock();
-      try {
-        for (TezEvent event : requestEvents) {
-          if (event.getEvent() instanceof TaskAttemptFailedEvent) {
-            fail("Found a TaskAttemptFailedEvent when not expected");
-          }
-          if (event.getEvent() instanceof TaskAttemptCompletedEvent) {
-            fail("Found a TaskAttemptCompletedvent when not expected");
-          }
-        }
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    public void verifyTaskFailedEvent(String diagnostics) {
-      umbilicalLock.lock();
-      try {
-        for (TezEvent event : requestEvents) {
-          if (event.getEvent() instanceof TaskAttemptFailedEvent) {
-            TaskAttemptFailedEvent failedEvent = (TaskAttemptFailedEvent)event.getEvent();
-            if(failedEvent.getDiagnostics().startsWith(diagnostics)){
-              return ;
-            } else {
-              fail("No detailed diagnostics message in TaskAttemptFailedEvent");
-            }
-          }
-        }
-        fail("No TaskAttemptFailedEvents sent over umbilical");
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    public void verifyTaskSuccessEvent() {
-      umbilicalLock.lock();
-      try {
-        for (TezEvent event : requestEvents) {
-          if (event.getEvent() instanceof TaskAttemptCompletedEvent) {
-            return;
-          }
-        }
-        fail("No TaskAttemptFailedEvents sent over umbilical");
-      } finally {
-        umbilicalLock.unlock();
-      }
-    }
-
-    @Override
-    public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
-      return 0;
-    }
-
-    @Override
-    public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
-        int clientMethodsHash) throws IOException {
-      return null;
-    }
-
-    @Override
-    public ContainerTask getTask(ContainerContext containerContext) throws IOException {
-      // Return shouldDie = true
-      getTaskInvocations++;
-      return new ContainerTask(null, true, null, null, false);
-    }
-
-    @Override
-    public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
-      return true;
-    }
-
-    @Override
-    public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
-        TezException {
-      umbilicalLock.lock();
-      if (request.getEvents() != null) {
-        requestEvents.addAll(request.getEvents());
-      }
-      try {
-        if (shouldThrowException) {
-          LOG.info("TestUmbilical throwing Exception");
-          throw new IOException(HEARTBEAT_EXCEPTION_STRING);
-        }
-        TezHeartbeatResponse response = new TezHeartbeatResponse();
-        response.setLastRequestId(request.getRequestId());
-        if (shouldSendDieSignal) {
-          LOG.info("TestUmbilical returning shouldDie=true");
-          response.setShouldDie();
-        }
-        return response;
-      } finally {
-        if (pendingEvent) {
-          eventEnacted = true;
-          LOG.info("Signalling Event");
-          eventCondition.signal();
-        }
-        umbilicalLock.unlock();
-      }
-    }
-  }
-
-  private TaskReporter createTaskReporter(ApplicationId appId, TezTaskUmbilicalForTest umbilical) {
-    TaskReporter taskReporter = new TaskReporter(umbilical, 100, 1000, 100, new AtomicLong(0),
-        createContainerId(appId).toString());
-    return taskReporter;
-  }
 
   private TezTaskRunner createTaskRunner(ApplicationId appId, TezTaskUmbilicalForTest umbilical,
       TaskReporter taskReporter, ListeningExecutorService executor, byte[] processorConf)
@@ -722,9 +358,5 @@ public class TestTaskExecution {
     return taskRunner;
   }
 
-  private ContainerId createContainerId(ApplicationId appId) {
-    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
-    ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
-    return containerId;
-  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
new file mode 100644
index 0000000..12d9d3f
--- /dev/null
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java
@@ -0,0 +1,638 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.task;
+
+import static org.apache.tez.runtime.task.TaskExecutionTestHelpers.createProcessorIOException;
+import static org.apache.tez.runtime.task.TaskExecutionTestHelpers.createProcessorTezException;
+import static org.apache.tez.runtime.task.TaskExecutionTestHelpers.createTaskReporter;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.dag.records.TezTaskID;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.api.ExecutionContext;
+import org.apache.tez.runtime.api.ObjectRegistry;
+import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
+import org.apache.tez.runtime.api.impl.InputSpec;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.common.resources.ScalingAllocator;
+import org.apache.tez.runtime.internals.api.TaskReporterInterface;
+import org.apache.tez.runtime.task.TaskExecutionTestHelpers.TestProcessor;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestTaskExecution2 {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestTaskExecution2.class);
+
+  private static final Configuration defaultConf = new Configuration();
+  private static final FileSystem localFs;
+  private static final Path workDir;
+
+  private static final ExecutorService taskExecutor = Executors.newFixedThreadPool(1);
+
+  static {
+    defaultConf.set("fs.defaultFS", "file:///");
+    defaultConf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS,
+        ScalingAllocator.class.getName());
+    try {
+      localFs = FileSystem.getLocal(defaultConf);
+      Path wd = new Path(System.getProperty("test.build.data", "/tmp"),
+          TestTaskExecution.class.getSimpleName());
+      workDir = localFs.makeQualified(wd);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Before
+  public void reset() {
+    TestProcessor.reset();
+  }
+
+  @AfterClass
+  public static void shutdown() {
+    taskExecutor.shutdownNow();
+  }
+
+  @Test(timeout = 5000)
+  public void testSingleSuccessfulTask() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(
+          new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.signal();
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false);
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskSuccessEvent();
+      assertFalse(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testMultipleSuccessfulTasks() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture = taskExecutor.submit(
+          new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.signal();
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false);
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskSuccessEvent();
+      assertFalse(TestProcessor.wasAborted());
+      umbilical.resetTrackedEvents();
+
+      taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      taskRunnerFuture = taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.signal();
+      result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false);
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskSuccessEvent();
+      assertFalse(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  // test task failed due to exception in Processor
+  @Test(timeout = 5000)
+  public void testFailedTaskTezException() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_THROW_TEZ_EXCEPTION);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+      TestProcessor.signal();
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorTezException(), false);
+
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskFailedEvent(
+          "Failure while running task",
+          TezException.class.getName() + ": " + TezException.class.getSimpleName());
+      // Failure detected as a result of fall off from the run method. abort isn't required.
+      assertFalse(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+
+  // Test task failed due to Processor class not found
+  @Test(timeout = 5000)
+  public void testFailedTask2() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          "NotExitedProcessor", TestProcessor.CONF_EMPTY, false);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.TASK_ERROR,
+          new TezUncheckedException("Unchecked exception"), false);
+
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskFailedEvent("Failure while running task",
+          ":org.apache.tez.dag.api.TezUncheckedException: "
+              + "Unable to load class: NotExitedProcessor");
+      // Failure detected as a result of fall off from the run method. abort isn't required.
+      assertFalse(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  // test task failed due to exception in Processor
+  @Test(timeout = 5000)
+  public void testFailedTaskIOException() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_THROW_IO_EXCEPTION);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+      TestProcessor.signal();
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false);
+
+
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskFailedEvent(
+          "Failure while running task",
+          IOException.class.getName() + ": " + IOException.class.getSimpleName());
+      // Failure detected as a result of fall off from the run method. abort isn't required.
+      assertFalse(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testHeartbeatException() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+      umbilical.signalThrowException();
+      umbilical.awaitRegisteredEvent();
+      // Not signaling an actual start to verify task interruption
+
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.COMMUNICATION_FAILURE,
+          new IOException("IOException"),
+          TaskExecutionTestHelpers.HEARTBEAT_EXCEPTION_STRING, false);
+
+      TestProcessor.awaitCompletion();
+      assertTrue(TestProcessor.wasInterrupted());
+      assertNull(taskReporter.currentCallable);
+      // No completion events since umbilical communication already failed.
+      umbilical.verifyNoCompletionEvents();
+      assertTrue(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testHeartbeatShouldDie() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+      umbilical.signalSendShouldDie();
+      umbilical.awaitRegisteredEvent();
+      // Not signaling an actual start to verify task interruption
+
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.CONTAINER_STOP_REQUESTED, null, true);
+
+
+      TestProcessor.awaitCompletion();
+      assertTrue(TestProcessor.wasInterrupted());
+      assertNull(taskReporter.currentCallable);
+      // TODO Is this statement correct ?
+      // No completion events since shouldDie was requested by the AM, which should have killed the
+      // task.
+      umbilical.verifyNoCompletionEvents();
+      assertTrue(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testSignalFatalErrorAndLoop() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_SIGNAL_FATAL_AND_LOOP);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+      TestProcessor.signal();
+
+      TestProcessor.awaitLoop();
+      // The fatal error should have caused an interrupt.
+
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false);
+
+      TestProcessor.awaitCompletion();
+      assertTrue(TestProcessor.wasInterrupted());
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskFailedEvent(
+          "Failure while running task",
+          IOException.class.getName() + ": " + IOException.class.getSimpleName());
+      // Signal fatal error should cause the processor to fail.
+      assertTrue(TestProcessor.wasAborted());
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testTaskKilled() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2 taskRunner = createTaskRunner(appId, umbilical, taskReporter, executor,
+          TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+
+      taskRunner.killTask();
+
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.KILL_REQUESTED, null, false);
+
+      TestProcessor.awaitCompletion();
+      assertTrue(TestProcessor.wasInterrupted());
+      assertNull(taskReporter.currentCallable);
+      // Kill events are not sent over the umbilical at the moment.
+      umbilical.verifyNoCompletionEvents();
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testKilledAfterComplete() throws IOException, InterruptedException, TezException,
+      ExecutionException {
+
+    ListeningExecutorService executor = null;
+    try {
+      ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
+      executor = MoreExecutors.listeningDecorator(rawExecutor);
+      ApplicationId appId = ApplicationId.newInstance(10000, 1);
+      TaskExecutionTestHelpers.TezTaskUmbilicalForTest
+          umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
+      TaskReporter taskReporter = createTaskReporter(appId, umbilical);
+
+      TezTaskRunner2ForTest taskRunner =
+          createTaskRunnerForTest(appId, umbilical, taskReporter, executor,
+              TestProcessor.CONF_EMPTY);
+      // Setup the executor
+      Future<TaskRunner2Result> taskRunnerFuture =
+          taskExecutor.submit(new TaskRunnerCallable2ForTest(taskRunner));
+      // Signal the processor to go through
+      TestProcessor.awaitStart();
+      TestProcessor.signal();
+      TestProcessor.awaitCompletion();
+
+      taskRunner.awaitCallableCompletion();
+
+      taskRunner.killTask();
+      TaskRunner2Result result = taskRunnerFuture.get();
+      verifyTaskRunnerResult(result, EndReason.SUCCESS, null, false);
+
+      assertFalse(TestProcessor.wasInterrupted());
+      assertNull(taskReporter.currentCallable);
+      umbilical.verifyTaskSuccessEvent();
+    } finally {
+      executor.shutdownNow();
+    }
+  }
+
+
+  private void verifyTaskRunnerResult(TaskRunner2Result taskRunner2Result,
+                                      EndReason expectedEndReason, Throwable expectedThrowable,
+                                      boolean wasShutdownRequested) {
+    verifyTaskRunnerResult(taskRunner2Result, expectedEndReason, expectedThrowable, null,
+        wasShutdownRequested);
+  }
+
+  private void verifyTaskRunnerResult(TaskRunner2Result taskRunner2Result,
+                                      EndReason expectedEndReason, Throwable expectedThrowable,
+                                      String expectedExceptionMessage,
+                                      boolean wasShutdownRequested) {
+    assertEquals(expectedEndReason, taskRunner2Result.getEndReason());
+    if (expectedThrowable == null) {
+      assertNull(taskRunner2Result.getError());
+    } else {
+      assertNotNull(taskRunner2Result.getError());
+      Throwable cause = taskRunner2Result.getError();
+      LOG.info(cause.getClass().getName());
+      assertTrue(cause.getClass().isAssignableFrom(expectedThrowable.getClass()));
+
+      if (expectedExceptionMessage != null) {
+        assertTrue(cause.getMessage().contains(expectedExceptionMessage));
+      }
+
+    }
+    assertEquals(wasShutdownRequested, taskRunner2Result.isContainerShutdownRequested());
+  }
+
+
+  private static class TaskRunnerCallable2ForTest implements Callable<TaskRunner2Result> {
+    private final TezTaskRunner2 taskRunner;
+
+    public TaskRunnerCallable2ForTest(TezTaskRunner2 taskRunner) {
+      this.taskRunner = taskRunner;
+    }
+
+    @Override
+    public TaskRunner2Result call() throws Exception {
+      return taskRunner.run();
+    }
+  }
+
+  private TezTaskRunner2 createTaskRunner(ApplicationId appId,
+                                          TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical,
+                                          TaskReporter taskReporter,
+                                          ListeningExecutorService executor, byte[] processorConf)
+      throws IOException {
+    return createTaskRunner(appId, umbilical, taskReporter, executor, TestProcessor.class.getName(),
+        processorConf, false);
+  }
+
+  private TezTaskRunner2ForTest createTaskRunnerForTest(ApplicationId appId,
+                                                        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical,
+                                                        TaskReporter taskReporter,
+                                                        ListeningExecutorService executor,
+                                                        byte[] processorConf)
+      throws IOException {
+    return (TezTaskRunner2ForTest) createTaskRunner(appId, umbilical, taskReporter, executor,
+        TestProcessor.class.getName(),
+        processorConf, true);
+  }
+
+  private TezTaskRunner2 createTaskRunner(ApplicationId appId,
+                                          TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical,
+                                          TaskReporter taskReporter,
+                                          ListeningExecutorService executor, String processorClass,
+                                          byte[] processorConf, boolean testRunner) throws
+      IOException {
+    TezConfiguration tezConf = new TezConfiguration(defaultConf);
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    Path testDir = new Path(workDir, UUID.randomUUID().toString());
+    String[] localDirs = new String[]{testDir.toString()};
+
+    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
+    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
+    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
+    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
+    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
+        .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
+    TaskSpec taskSpec =
+        new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor,
+            new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);
+
+    TezTaskRunner2 taskRunner;
+    if (testRunner) {
+      taskRunner = new TezTaskRunner2ForTest(tezConf, ugi, localDirs, taskSpec, 1,
+          new HashMap<String, ByteBuffer>(), new HashMap<String, String>(),
+          HashMultimap.<String, String>create(), taskReporter,
+          executor, null, "", new ExecutionContextImpl("localhost"),
+          Runtime.getRuntime().maxMemory());
+    } else {
+      taskRunner = new TezTaskRunner2(tezConf, ugi, localDirs, taskSpec, 1,
+          new HashMap<String, ByteBuffer>(), new HashMap<String, String>(),
+          HashMultimap.<String, String>create(), taskReporter,
+          executor, null, "", new ExecutionContextImpl("localhost"),
+          Runtime.getRuntime().maxMemory());
+    }
+
+    return taskRunner;
+  }
+
+  public static class TezTaskRunner2ForTest extends TezTaskRunner2 {
+
+    private final ReentrantLock testLock = new ReentrantLock();
+    private final Condition callableCompletionCondition = testLock.newCondition();
+
+    private final AtomicBoolean isCallableComplete = new AtomicBoolean(false);
+
+    public TezTaskRunner2ForTest(Configuration tezConf, UserGroupInformation ugi,
+                                 String[] localDirs,
+                                 TaskSpec taskSpec, int appAttemptNumber,
+                                 Map<String, ByteBuffer> serviceConsumerMetadata,
+                                 Map<String, String> serviceProviderEnvMap,
+                                 Multimap<String, String> startedInputsMap,
+                                 TaskReporterInterface taskReporter,
+                                 ListeningExecutorService executor,
+                                 ObjectRegistry objectRegistry,
+                                 String pid,
+                                 ExecutionContext executionContext,
+                                 long memAvailable) throws IOException {
+      super(tezConf, ugi, localDirs, taskSpec, appAttemptNumber, serviceConsumerMetadata,
+          serviceProviderEnvMap, startedInputsMap, taskReporter, executor, objectRegistry, pid,
+          executionContext, memAvailable);
+    }
+
+
+    @Override
+    @VisibleForTesting
+    void processCallableResult(TaskRunner2Callable.TaskRunner2CallableResult executionResult) {
+      testLock.lock();
+      try {
+        super.processCallableResult(executionResult);
+        isCallableComplete.set(true);
+        callableCompletionCondition.signal();
+      } finally {
+        testLock.unlock();
+      }
+    }
+
+    void awaitCallableCompletion() throws InterruptedException {
+      testLock.lock();
+      try {
+        while (!isCallableComplete.get()) {
+          callableCompletionCondition.await();
+        }
+      } finally {
+        testLock.unlock();
+      }
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/509238b7/tez-runtime-internals/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/resources/log4j.properties b/tez-runtime-internals/src/test/resources/log4j.properties
new file mode 100644
index 0000000..531b68b
--- /dev/null
+++ b/tez-runtime-internals/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n


[2/2] tez git commit: TEZ-2657. Add tests for client side changes - specifying plugins, etc. (sseth)

Posted by ss...@apache.org.
TEZ-2657. Add tests for client side changes - specifying plugins, etc. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a169d8ae
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a169d8ae
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a169d8ae

Branch: refs/heads/TEZ-2003
Commit: a169d8ae9f191c93b9c53873c0e44f2f8ecd6b05
Parents: 509238b
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Jul 29 18:26:01 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Jul 29 18:26:01 2015 -0700

----------------------------------------------------------------------
 TEZ-2003-CHANGES.txt                            |   1 +
 .../java/org/apache/tez/client/TezClient.java   |  15 +-
 .../org/apache/tez/client/TezClientUtils.java   |  38 +---
 .../apache/tez/dag/api/DagTypeConverters.java   |  67 +++++--
 .../java/org/apache/tez/dag/api/Vertex.java     |  41 ++++
 .../api/ServicePluginsDescriptor.java           |  36 ++++
 tez-api/src/main/proto/DAGApiRecords.proto      |   2 +-
 .../org/apache/tez/client/TestTezClient.java    | 113 +++++++++--
 .../apache/tez/client/TestTezClientUtils.java   |  16 +-
 .../org/apache/tez/dag/api/TestDAGPlan.java     |  63 +++++-
 .../tez/dag/api/TestDagTypeConverters.java      | 196 ++++++++++++++++++-
 .../org/apache/tez/dag/app/DAGAppMaster.java    |   4 +-
 12 files changed, 508 insertions(+), 84 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/TEZ-2003-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2003-CHANGES.txt b/TEZ-2003-CHANGES.txt
index 9d72d92..9b3967a 100644
--- a/TEZ-2003-CHANGES.txt
+++ b/TEZ-2003-CHANGES.txt
@@ -40,5 +40,6 @@ ALL CHANGES:
   TEZ-2652. Cleanup the way services are specified for an AM and vertices.
   TEZ-2653. Change service contexts to expose a user specified payload instead of the AM configuration.
   TEZ-2441. Add tests for TezTaskRunner2.
+  TEZ-2657. Add tests for client side changes - specifying plugins, etc.
 
 INCOMPATIBLE CHANGES:

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index f961291..7778ef3 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -94,13 +94,16 @@ public class TezClient {
   @VisibleForTesting
   static final String NO_CLUSTER_DIAGNOSTICS_MSG = "No cluster diagnostics found.";
 
-  private final String clientName;
+  @VisibleForTesting
+  final String clientName;
   private ApplicationId sessionAppId;
   private ApplicationId lastSubmittedAppId;
-  private AMConfiguration amConfig;
+  @VisibleForTesting
+  final AMConfiguration amConfig;
   private FrameworkClient frameworkClient;
   private String diagnostics;
-  private boolean isSession;
+  @VisibleForTesting
+  final boolean isSession;
   private boolean sessionStarted = false;
   private boolean sessionStopped = false;
   /** Tokens which will be required for all DAGs submitted to this session. */
@@ -112,8 +115,10 @@ public class TezClient {
   private JobTokenSecretManager jobTokenSecretManager =
       new JobTokenSecretManager();
   private final Map<String, LocalResource> additionalLocalResources = Maps.newHashMap();
-  private final TezApiVersionInfo apiVersionInfo;
-  private final ServicePluginsDescriptor servicePluginsDescriptor;
+  @VisibleForTesting
+  final TezApiVersionInfo apiVersionInfo;
+  @VisibleForTesting
+  final ServicePluginsDescriptor servicePluginsDescriptor;
   private HistoryACLPolicyManager historyACLPolicyManager;
 
   private int preWarmDAGCounter = 0;

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 60e1305..d161392 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -39,9 +39,7 @@ import java.util.Map.Entry;
 
 import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
-import org.apache.tez.dag.api.NamedEntityDescriptor;
 import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
-import org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto;
 import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -775,47 +773,13 @@ public class TezClientUtils {
     }
 
     AMPluginDescriptorProto pluginDescriptorProto =
-        createAMServicePluginDescriptorProto(servicePluginsDescriptor);
+        DagTypeConverters.convertServicePluginDescriptoToProto(servicePluginsDescriptor);
     builder.setAmPluginDescriptor(pluginDescriptorProto);
 
     return builder.build();
   }
 
-  static AMPluginDescriptorProto createAMServicePluginDescriptorProto(
-      ServicePluginsDescriptor servicePluginsDescriptor) {
-    AMPluginDescriptorProto.Builder pluginDescriptorBuilder =
-        AMPluginDescriptorProto.newBuilder();
-    if (servicePluginsDescriptor != null) {
 
-      pluginDescriptorBuilder.setContainersEnabled(servicePluginsDescriptor.areContainersEnabled());
-      pluginDescriptorBuilder.setUberEnabled(servicePluginsDescriptor.isUberEnabled());
-
-      if (servicePluginsDescriptor.getTaskSchedulerDescriptors() != null &&
-          servicePluginsDescriptor.getTaskSchedulerDescriptors().length > 0) {
-        List<TezNamedEntityDescriptorProto> namedEntityProtos = DagTypeConverters.convertNamedEntityCollectionToProto(
-            servicePluginsDescriptor.getTaskSchedulerDescriptors());
-        pluginDescriptorBuilder.addAllTaskScedulers(namedEntityProtos);
-      }
-
-      if (servicePluginsDescriptor.getContainerLauncherDescriptors() != null &&
-          servicePluginsDescriptor.getContainerLauncherDescriptors().length > 0) {
-        List<TezNamedEntityDescriptorProto> namedEntityProtos = DagTypeConverters.convertNamedEntityCollectionToProto(
-            servicePluginsDescriptor.getContainerLauncherDescriptors());
-        pluginDescriptorBuilder.addAllContainerLaunchers(namedEntityProtos);
-      }
-
-      if (servicePluginsDescriptor.getTaskCommunicatorDescriptors() != null &&
-          servicePluginsDescriptor.getTaskCommunicatorDescriptors().length > 0) {
-        List<TezNamedEntityDescriptorProto> namedEntityProtos = DagTypeConverters.convertNamedEntityCollectionToProto(
-            servicePluginsDescriptor.getTaskCommunicatorDescriptors());
-        pluginDescriptorBuilder.addAllTaskCommunicators(namedEntityProtos);
-      }
-
-    } else {
-      pluginDescriptorBuilder.setContainersEnabled(true).setUberEnabled(false);
-    }
-    return pluginDescriptorBuilder.build();
-  }
 
   /**
    * Helper function to create a YARN LocalResource

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index 2e0d417..61e4d33 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -52,9 +52,11 @@ import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
 import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
 import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
+import org.apache.tez.dag.api.Vertex.VertexExecutionContext;
 import org.apache.tez.dag.api.client.StatusGetOpts;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TezAppMasterStatusProto;
 import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
 import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
 import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
 import org.apache.tez.dag.api.records.DAGProtos.PlanEdgeDataMovementType;
@@ -74,14 +76,13 @@ import org.apache.tez.dag.api.records.DAGProtos.TezCounterProto;
 import org.apache.tez.dag.api.records.DAGProtos.TezCountersProto;
 import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
 import org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto;
+import org.apache.tez.dag.api.records.DAGProtos.VertexExecutionContextProto;
 import org.apache.tez.dag.api.records.DAGProtos.VertexLocationHintProto;
 
 import com.google.common.collect.Lists;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.ByteString.Output;
-import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor;
-import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor;
-import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor;
+import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
 
 @Private
 public class DagTypeConverters {
@@ -732,13 +733,13 @@ public class DagTypeConverters {
     return payload.getPayload();
   }
 
-  public static DAGProtos.VertexExecutionContextProto convertToProto(
-      Vertex.VertexExecutionContext context) {
+  public static VertexExecutionContextProto convertToProto(
+      VertexExecutionContext context) {
     if (context == null) {
       return null;
     } else {
-      DAGProtos.VertexExecutionContextProto.Builder builder =
-          DAGProtos.VertexExecutionContextProto.newBuilder();
+      VertexExecutionContextProto.Builder builder =
+          VertexExecutionContextProto.newBuilder();
       builder.setExecuteInAm(context.shouldExecuteInAm());
       builder.setExecuteInContainers(context.shouldExecuteInContainers());
       if (context.getTaskSchedulerName() != null) {
@@ -754,26 +755,26 @@ public class DagTypeConverters {
     }
   }
 
-  public static Vertex.VertexExecutionContext convertFromProto(
-      DAGProtos.VertexExecutionContextProto proto) {
+  public static VertexExecutionContext convertFromProto(
+      VertexExecutionContextProto proto) {
     if (proto == null) {
       return null;
     } else {
       if (proto.getExecuteInAm()) {
-        Vertex.VertexExecutionContext context =
-            Vertex.VertexExecutionContext.createExecuteInAm(proto.getExecuteInAm());
+        VertexExecutionContext context =
+            VertexExecutionContext.createExecuteInAm(proto.getExecuteInAm());
         return context;
       } else if (proto.getExecuteInContainers()) {
-        Vertex.VertexExecutionContext context =
-            Vertex.VertexExecutionContext.createExecuteInContainers(proto.getExecuteInContainers());
+        VertexExecutionContext context =
+            VertexExecutionContext.createExecuteInContainers(proto.getExecuteInContainers());
         return context;
       } else {
         String taskScheduler = proto.hasTaskSchedulerName() ? proto.getTaskSchedulerName() : null;
         String containerLauncher =
             proto.hasContainerLauncherName() ? proto.getContainerLauncherName() : null;
         String taskComm = proto.hasTaskCommName() ? proto.getTaskCommName() : null;
-        Vertex.VertexExecutionContext context =
-            Vertex.VertexExecutionContext.create(taskScheduler, containerLauncher, taskComm);
+        VertexExecutionContext context =
+            VertexExecutionContext.create(taskScheduler, containerLauncher, taskComm);
         return context;
       }
     }
@@ -800,4 +801,40 @@ public class DagTypeConverters {
     return builder.build();
   }
 
+  public static AMPluginDescriptorProto convertServicePluginDescriptoToProto(
+      ServicePluginsDescriptor servicePluginsDescriptor) {
+    AMPluginDescriptorProto.Builder pluginDescriptorBuilder =
+        AMPluginDescriptorProto.newBuilder();
+    if (servicePluginsDescriptor != null) {
+
+      pluginDescriptorBuilder.setContainersEnabled(servicePluginsDescriptor.areContainersEnabled());
+      pluginDescriptorBuilder.setUberEnabled(servicePluginsDescriptor.isUberEnabled());
+
+      if (servicePluginsDescriptor.getTaskSchedulerDescriptors() != null &&
+          servicePluginsDescriptor.getTaskSchedulerDescriptors().length > 0) {
+        List<TezNamedEntityDescriptorProto> namedEntityProtos = DagTypeConverters.convertNamedEntityCollectionToProto(
+            servicePluginsDescriptor.getTaskSchedulerDescriptors());
+        pluginDescriptorBuilder.addAllTaskSchedulers(namedEntityProtos);
+      }
+
+      if (servicePluginsDescriptor.getContainerLauncherDescriptors() != null &&
+          servicePluginsDescriptor.getContainerLauncherDescriptors().length > 0) {
+        List<TezNamedEntityDescriptorProto> namedEntityProtos = DagTypeConverters.convertNamedEntityCollectionToProto(
+            servicePluginsDescriptor.getContainerLauncherDescriptors());
+        pluginDescriptorBuilder.addAllContainerLaunchers(namedEntityProtos);
+      }
+
+      if (servicePluginsDescriptor.getTaskCommunicatorDescriptors() != null &&
+          servicePluginsDescriptor.getTaskCommunicatorDescriptors().length > 0) {
+        List<TezNamedEntityDescriptorProto> namedEntityProtos = DagTypeConverters.convertNamedEntityCollectionToProto(
+            servicePluginsDescriptor.getTaskCommunicatorDescriptors());
+        pluginDescriptorBuilder.addAllTaskCommunicators(namedEntityProtos);
+      }
+
+    } else {
+      pluginDescriptorBuilder.setContainersEnabled(true).setUberEnabled(false);
+    }
+    return pluginDescriptorBuilder.build();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 34124b2..8953ae1 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -511,6 +511,47 @@ public class Vertex {
           ", taskCommName='" + taskCommName + '\'' +
           '}';
     }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+
+      VertexExecutionContext that = (VertexExecutionContext) o;
+
+      if (executeInAm != that.executeInAm) {
+        return false;
+      }
+      if (executeInContainers != that.executeInContainers) {
+        return false;
+      }
+      if (taskSchedulerName != null ? !taskSchedulerName.equals(that.taskSchedulerName) :
+          that.taskSchedulerName != null) {
+        return false;
+      }
+      if (containerLauncherName != null ?
+          !containerLauncherName.equals(that.containerLauncherName) :
+          that.containerLauncherName != null) {
+        return false;
+      }
+      return !(taskCommName != null ? !taskCommName.equals(that.taskCommName) :
+          that.taskCommName != null);
+
+    }
+
+    @Override
+    public int hashCode() {
+      int result = (executeInAm ? 1 : 0);
+      result = 31 * result + (executeInContainers ? 1 : 0);
+      result = 31 * result + (taskSchedulerName != null ? taskSchedulerName.hashCode() : 0);
+      result = 31 * result + (containerLauncherName != null ? containerLauncherName.hashCode() : 0);
+      result = 31 * result + (taskCommName != null ? taskCommName.hashCode() : 0);
+      return result;
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
index 8df102a..2e4fc46 100644
--- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java
@@ -46,6 +46,15 @@ public class ServicePluginsDescriptor {
     this.taskCommunicatorDescriptors = taskCommunicatorDescriptors;
   }
 
+  /**
+   * Create a service plugin descriptor with the provided plugins. Regular containers will also be enabled
+   * when using this method.
+   *
+   * @param taskSchedulerDescriptor the task scheduler plugin descriptors
+   * @param containerLauncherDescriptors the container launcher plugin descriptors
+   * @param taskCommunicatorDescriptors the task communicator plugin descriptors
+   * @return
+   */
   public static ServicePluginsDescriptor create(TaskSchedulerDescriptor[] taskSchedulerDescriptor,
                                                 ContainerLauncherDescriptor[] containerLauncherDescriptors,
                                                 TaskCommunicatorDescriptor[] taskCommunicatorDescriptors) {
@@ -53,6 +62,15 @@ public class ServicePluginsDescriptor {
         containerLauncherDescriptors, taskCommunicatorDescriptors);
   }
 
+  /**
+   * Create a service plugin descriptor with the provided plugins. Also allows specification of whether
+   * in-AM execution is enabled. Container execution is enabled by default.
+   * @param enableUber whether to enable execution in the AM or not
+   * @param taskSchedulerDescriptor the task scheduler plugin descriptors
+   * @param containerLauncherDescriptors the container launcher plugin descriptors
+   * @param taskCommunicatorDescriptors the task communicator plugin descriptors
+   * @return
+   */
   public static ServicePluginsDescriptor create(boolean enableUber,
                                                 TaskSchedulerDescriptor[] taskSchedulerDescriptor,
                                                 ContainerLauncherDescriptor[] containerLauncherDescriptors,
@@ -61,6 +79,17 @@ public class ServicePluginsDescriptor {
         containerLauncherDescriptors, taskCommunicatorDescriptors);
   }
 
+  /**
+   * Create a service plugin descriptor with the provided plugins. Also allows specification of whether
+   * container execution and in-AM execution will be enabled.
+   *
+   * @param enableContainers whether to enable execution in containers
+   * @param enableUber whether to enable execution in the AM or not
+   * @param taskSchedulerDescriptor the task scheduler plugin descriptors
+   * @param containerLauncherDescriptors the container launcher plugin descriptors
+   * @param taskCommunicatorDescriptors the task communicator plugin descriptors
+   * @return
+   */
   public static ServicePluginsDescriptor create(boolean enableContainers, boolean enableUber,
                                                 TaskSchedulerDescriptor[] taskSchedulerDescriptor,
                                                 ContainerLauncherDescriptor[] containerLauncherDescriptors,
@@ -69,6 +98,13 @@ public class ServicePluginsDescriptor {
         containerLauncherDescriptors, taskCommunicatorDescriptors);
   }
 
+  /**
+   * Create a service plugin descriptor which may have in-AM execution of tasks enabled. Container
+   * execution is enabled by default
+   *
+   * @param enableUber whether to enable execution in the AM or not
+   * @return
+   */
   public static ServicePluginsDescriptor create(boolean enableUber) {
     return new ServicePluginsDescriptor(true, enableUber, null, null, null);
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/DAGApiRecords.proto b/tez-api/src/main/proto/DAGApiRecords.proto
index ebe3259..193f7b8 100644
--- a/tez-api/src/main/proto/DAGApiRecords.proto
+++ b/tez-api/src/main/proto/DAGApiRecords.proto
@@ -180,7 +180,7 @@ message TezNamedEntityDescriptorProto {
 message AMPluginDescriptorProto {
   optional bool containers_enabled = 1 [default = true];
   optional bool uber_enabled = 2 [default = false];
-  repeated TezNamedEntityDescriptorProto task_scedulers = 3;
+  repeated TezNamedEntityDescriptorProto task_schedulers = 3;
   repeated TezNamedEntityDescriptorProto container_launchers = 4;
   repeated TezNamedEntityDescriptorProto task_communicators = 5;
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index dc0fbb1..7a642e6 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -26,6 +26,11 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import javax.annotation.Nullable;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.mock;
@@ -61,6 +66,7 @@ import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAMStatusRespo
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.ShutdownSessionRequestProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TezAppMasterStatusProto;
+import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
 import org.hamcrest.CoreMatchers;
 import org.junit.Assert;
 import org.junit.Test;
@@ -153,11 +159,11 @@ public class TestTezClient {
       verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
       ApplicationSubmissionContext context = captor.getValue();
       Assert.assertEquals(3, context.getAMContainerSpec().getLocalResources().size());
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_PB_BINARY_CONF_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           lrName1));
     } else {
       verify(client.mockYarnClient, times(0)).submitApplication(captor.capture());
@@ -172,7 +178,7 @@ public class TestTezClient {
     DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG);
     DAGClient dagClient = client.submitDAG(dag);
         
-    Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
+    assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
     
     if (isSession) {
       verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
@@ -181,13 +187,13 @@ public class TestTezClient {
       verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
       ApplicationSubmissionContext context = captor.getValue();
       Assert.assertEquals(4, context.getAMContainerSpec().getLocalResources().size());
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_PB_BINARY_CONF_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_PB_PLAN_BINARY_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           lrName1));
     }
     
@@ -211,7 +217,7 @@ public class TestTezClient {
     if (isSession) {
       // same app master
       verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
-      Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
+      assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
       // additional resource is sent
       ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
       verify(client.sessionAmProxy, times(2)).submitDAG((RpcController)any(), captor1.capture());
@@ -220,20 +226,20 @@ public class TestTezClient {
       Assert.assertEquals(lrName2, proto.getAdditionalAmResources().getLocalResources(0).getName());
     } else {
       // new app master
-      Assert.assertTrue(dagClient.getExecutionContext().contains(appId2.toString()));
+      assertTrue(dagClient.getExecutionContext().contains(appId2.toString()));
       verify(client.mockYarnClient, times(2)).submitApplication(captor.capture());
       // additional resource is added
       ApplicationSubmissionContext context = captor.getValue();
       Assert.assertEquals(5, context.getAMContainerSpec().getLocalResources().size());
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_PB_BINARY_CONF_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           TezConstants.TEZ_PB_PLAN_BINARY_NAME));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           lrName1));
-      Assert.assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
+      assertTrue(context.getAMContainerSpec().getLocalResources().containsKey(
           lrName2));
     }
     
@@ -263,7 +269,7 @@ public class TestTezClient {
     ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
     verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), captor1.capture());
     SubmitDAGRequestProto proto = captor1.getValue();
-    Assert.assertTrue(proto.getDAGPlan().getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX));
+    assertTrue(proto.getDAGPlan().getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX));
 
     client.stop();
   }
@@ -330,7 +336,7 @@ public class TestTezClient {
     thread.join(250);
     thread.interrupt();
     thread.join();
-    Assert.assertThat(exceptionReference.get(),CoreMatchers. instanceOf(InterruptedException.class));
+    Assert.assertThat(exceptionReference.get(), CoreMatchers.instanceOf(InterruptedException.class));
     client.stop();
   }
   
@@ -347,7 +353,7 @@ public class TestTezClient {
       client.waitTillReady();
       Assert.fail();
     } catch (SessionNotRunning e) {
-      Assert.assertTrue(e.getMessage().contains(msg));
+      assertTrue(e.getMessage().contains(msg));
     }
     client.stop();
   }
@@ -362,7 +368,7 @@ public class TestTezClient {
       client.waitTillReady();
       Assert.fail();
     } catch (SessionNotRunning e) {
-      Assert.assertTrue(e.getMessage().contains(TezClient.NO_CLUSTER_DIAGNOSTICS_MSG));
+      assertTrue(e.getMessage().contains(TezClient.NO_CLUSTER_DIAGNOSTICS_MSG));
     }
     client.stop();
   }
@@ -387,9 +393,76 @@ public class TestTezClient {
       client.submitDAG(dag);
       Assert.fail();
     } catch (SessionNotRunning e) {
-      Assert.assertTrue(e.getMessage().contains(msg));
+      assertTrue(e.getMessage().contains(msg));
     }
     client.stop();
   }
 
+  @Test(timeout = 5000)
+  public void testClientBuilder() {
+    TezConfiguration tezConfWitSession = new TezConfiguration();
+    tezConfWitSession.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
+
+    TezConfiguration tezConfNoSession = new TezConfiguration();
+    tezConfNoSession.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
+
+    AMConfiguration amConf;
+    TezClient tezClient;
+    Credentials credentials = new Credentials();
+    Map<String, LocalResource> localResourceMap = new HashMap<>();
+    localResourceMap.put("testResource", mock(LocalResource.class));
+    ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
+
+    // Session mode via conf
+    tezClient = TezClient.newBuilder("client", tezConfWitSession).build();
+    assertTrue(tezClient.isSession);
+    assertNull(tezClient.servicePluginsDescriptor);
+    assertNotNull(tezClient.apiVersionInfo);
+    amConf = tezClient.amConfig;
+    assertNotNull(amConf);
+    assertEquals(0, amConf.getAMLocalResources().size());
+    assertNull(amConf.getCredentials());
+    assertTrue(
+        amConf.getTezConfiguration().getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false));
+
+    // Non-Session mode via conf
+    tezClient = TezClient.newBuilder("client", tezConfNoSession).build();
+    assertFalse(tezClient.isSession);
+    assertNull(tezClient.servicePluginsDescriptor);
+    assertNotNull(tezClient.apiVersionInfo);
+    amConf = tezClient.amConfig;
+    assertNotNull(amConf);
+    assertEquals(0, amConf.getAMLocalResources().size());
+    assertNull(amConf.getCredentials());
+    assertFalse(
+        amConf.getTezConfiguration().getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true));
+
+    // no-session via config. API explicit session.
+    tezClient = TezClient.newBuilder("client", tezConfNoSession).setIsSession(true).build();
+    assertTrue(tezClient.isSession);
+    assertNull(tezClient.servicePluginsDescriptor);
+    assertNotNull(tezClient.apiVersionInfo);
+    amConf = tezClient.amConfig;
+    assertNotNull(amConf);
+    assertEquals(0, amConf.getAMLocalResources().size());
+    assertNull(amConf.getCredentials());
+    assertTrue(
+        amConf.getTezConfiguration().getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false));
+
+    // Plugins, credentials, local resources
+    tezClient = TezClient.newBuilder("client", tezConfWitSession).setCredentials(credentials)
+        .setLocalResources(localResourceMap).setServicePluginDescriptor(servicePluginsDescriptor)
+        .build();
+    assertTrue(tezClient.isSession);
+    assertEquals(servicePluginsDescriptor, tezClient.servicePluginsDescriptor);
+    assertNotNull(tezClient.apiVersionInfo);
+    amConf = tezClient.amConfig;
+    assertNotNull(amConf);
+    assertEquals(1, amConf.getAMLocalResources().size());
+    assertEquals(localResourceMap, amConf.getAMLocalResources());
+    assertEquals(credentials, amConf.getCredentials());
+    assertTrue(
+        amConf.getTezConfiguration().getBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false));
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 4d211ba..01929ee 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -69,6 +69,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
 import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
+import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -464,7 +465,8 @@ public class TestTezClientUtils {
     Assert.assertNotNull(javaOpts);
     Assert.assertTrue(javaOpts.contains("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=FOOBAR")
         && javaOpts.contains(TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE)
-        && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"));
+        &&
+        javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"));
   }
 
   @Test (timeout = 5000)
@@ -641,6 +643,16 @@ public class TestTezClientUtils {
     Assert.assertTrue(resourceNames.contains("dir2-f.txt"));
   }
 
-  // TODO TEZ-2003 Add test to validate ServicePluginDescriptor propagation
+  @Test(timeout = 5000)
+  public void testServiceDescriptorSerializationForAM() {
+    Configuration conf = new Configuration(false);
+    ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
+
+    ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, null,
+        servicePluginsDescriptor);
+
+    assertTrue(confProto.hasAmPluginDescriptor());
+    assertTrue(confProto.getAmPluginDescriptor().getUberEnabled());
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index fccbb08..cd42109 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -37,11 +37,14 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
 import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
 import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
+import org.apache.tez.dag.api.Vertex.VertexExecutionContext;
+import org.apache.tez.dag.api.records.DAGProtos;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
 import org.apache.tez.dag.api.records.DAGProtos.PlanTaskConfiguration;
 import org.apache.tez.dag.api.records.DAGProtos.PlanTaskLocationHint;
 import org.apache.tez.dag.api.records.DAGProtos.PlanVertexType;
+import org.apache.tez.dag.api.records.DAGProtos.VertexExecutionContextProto;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
 import org.junit.Assert;
 import org.junit.Rule;
@@ -131,7 +134,8 @@ public class TestDAGPlan {
     EdgeManagerPluginDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
     Assert.assertNotNull(emDesc);
     Assert.assertEquals("emClass", emDesc.getClassName());
-    Assert.assertTrue(Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload().deepCopyAsArray()));
+    Assert.assertTrue(
+        Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload().deepCopyAsArray()));
   }
 
   @Test(timeout = 5000)
@@ -311,4 +315,61 @@ public class TestDAGPlan {
     assertNotNull(fetchedCredentials.getToken(new Text("Token1")));
     assertNotNull(fetchedCredentials.getToken(new Text("Token2")));
   }
+
+  @Test(timeout = 5000)
+  public void testServiceDescriptorPropagation() {
+    DAG dag = DAG.create("testDag");
+    ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").
+        setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+    ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").
+        setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
+
+    VertexExecutionContext defaultExecutionContext =
+        VertexExecutionContext.create("plugin", "plugin", "plugin");
+    VertexExecutionContext v1Context = VertexExecutionContext.createExecuteInAm(true);
+
+
+    Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1)).setExecutionContext(v1Context);
+    Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
+    v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
+        .addTaskLocalFiles(new HashMap<String, LocalResource>());
+    v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
+        .addTaskLocalFiles(new HashMap<String, LocalResource>());
+
+    InputDescriptor inputDescriptor = InputDescriptor.create("input").
+        setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+    OutputDescriptor outputDescriptor = OutputDescriptor.create("output").
+        setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
+    Edge edge = Edge.create(v1, v2, EdgeProperty.create(
+        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+        SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
+
+    dag.addVertex(v1).addVertex(v2).addEdge(edge);
+    dag.setExecutionContext(defaultExecutionContext);
+
+    DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);
+
+    assertEquals(2, dagProto.getVertexCount());
+    assertEquals(1, dagProto.getEdgeCount());
+
+    assertTrue(dagProto.hasDefaultExecutionContext());
+    VertexExecutionContextProto defaultContextProto = dagProto.getDefaultExecutionContext();
+    assertFalse(defaultContextProto.getExecuteInContainers());
+    assertFalse(defaultContextProto.getExecuteInAm());
+    assertEquals("plugin", defaultContextProto.getTaskSchedulerName());
+    assertEquals("plugin", defaultContextProto.getContainerLauncherName());
+    assertEquals("plugin", defaultContextProto.getTaskCommName());
+
+    VertexPlan v1Proto = dagProto.getVertex(0);
+    assertTrue(v1Proto.hasExecutionContext());
+    VertexExecutionContextProto v1ContextProto = v1Proto.getExecutionContext();
+    assertFalse(v1ContextProto.getExecuteInContainers());
+    assertTrue(v1ContextProto.getExecuteInAm());
+    assertFalse(v1ContextProto.hasTaskSchedulerName());
+    assertFalse(v1ContextProto.hasContainerLauncherName());
+    assertFalse(v1ContextProto.hasTaskCommName());
+
+    VertexPlan v2Proto = dagProto.getVertex(1);
+    assertFalse(v2Proto.hasExecutionContext());
+  }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
index 51b179a..e37f849 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
@@ -18,15 +18,32 @@
 
 package org.apache.tez.dag.api;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 import java.io.IOException;
 
 import java.nio.ByteBuffer;
+import java.util.List;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.dag.api.Vertex.VertexExecutionContext;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
 import org.apache.tez.dag.api.records.DAGProtos.TezEntityDescriptorProto;
+import org.apache.tez.dag.api.records.DAGProtos.TezNamedEntityDescriptorProto;
+import org.apache.tez.dag.api.records.DAGProtos.VertexExecutionContextProto;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor;
+import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
+import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -43,7 +60,7 @@ public class TestDagTypeConverters {
         DagTypeConverters.convertToDAGPlan(entityDescriptor);
     Assert.assertEquals(payload.getVersion(), proto.getTezUserPayload().getVersion());
     Assert.assertArrayEquals(payload.deepCopyAsArray(), proto.getTezUserPayload().getUserPayload().toByteArray());
-    Assert.assertTrue(proto.hasHistoryText());
+    assertTrue(proto.hasHistoryText());
     Assert.assertNotEquals(historytext, proto.getHistoryText());
     Assert.assertEquals(historytext, new String(
         TezCommonUtils.decompressByteStringToByteArray(proto.getHistoryText())));
@@ -89,4 +106,181 @@ public class TestDagTypeConverters {
     Assert.assertEquals(2311, lr2UrlDeserialized.getPort());
   }
 
+
+  @Test(timeout = 5000)
+  public void testVertexExecutionContextTranslation() {
+    VertexExecutionContext originalContext;
+    VertexExecutionContextProto contextProto;
+    VertexExecutionContext retrievedContext;
+
+
+    // Uber
+    originalContext = VertexExecutionContext.createExecuteInAm(true);
+    contextProto = DagTypeConverters.convertToProto(originalContext);
+    retrievedContext = DagTypeConverters.convertFromProto(contextProto);
+    assertEquals(originalContext, retrievedContext);
+
+    // Regular containers
+    originalContext = VertexExecutionContext.createExecuteInContainers(true);
+    contextProto = DagTypeConverters.convertToProto(originalContext);
+    retrievedContext = DagTypeConverters.convertFromProto(contextProto);
+    assertEquals(originalContext, retrievedContext);
+
+    // Custom
+    originalContext = VertexExecutionContext.create("plugin", "plugin", "plugin");
+    contextProto = DagTypeConverters.convertToProto(originalContext);
+    retrievedContext = DagTypeConverters.convertFromProto(contextProto);
+    assertEquals(originalContext, retrievedContext);
+  }
+
+
+  static final String testScheduler = "testScheduler";
+  static final String testLauncher = "testLauncher";
+  static final String testComm = "testComm";
+  static final String classSuffix = "_class";
+
+  @Test(timeout = 5000)
+  public void testServiceDescriptorTranslation() {
+
+
+    TaskSchedulerDescriptor[] taskSchedulers;
+    ContainerLauncherDescriptor[] containerLaunchers;
+    TaskCommunicatorDescriptor[] taskComms;
+
+    ServicePluginsDescriptor servicePluginsDescriptor;
+    AMPluginDescriptorProto proto;
+
+    // Uber-execution
+    servicePluginsDescriptor = ServicePluginsDescriptor.create(true);
+    proto = DagTypeConverters.convertServicePluginDescriptoToProto(servicePluginsDescriptor);
+    assertTrue(proto.hasUberEnabled());
+    assertTrue(proto.hasContainersEnabled());
+    assertTrue(proto.getUberEnabled());
+    assertTrue(proto.getContainersEnabled());
+    assertEquals(0, proto.getTaskSchedulersCount());
+    assertEquals(0, proto.getContainerLaunchersCount());
+    assertEquals(0, proto.getTaskCommunicatorsCount());
+
+    // Single plugin set specified. One with a payload.
+    taskSchedulers = createTaskScheduelrs(1, false);
+    containerLaunchers = createContainerLaunchers(1, false);
+    taskComms = createTaskCommunicators(1, true);
+
+    servicePluginsDescriptor = ServicePluginsDescriptor.create(taskSchedulers, containerLaunchers,
+        taskComms);
+    proto = DagTypeConverters.convertServicePluginDescriptoToProto(servicePluginsDescriptor);
+    assertTrue(proto.hasUberEnabled());
+    assertTrue(proto.hasContainersEnabled());
+    assertFalse(proto.getUberEnabled());
+    assertTrue(proto.getContainersEnabled());
+    verifyPlugins(proto.getTaskSchedulersList(), 1, testScheduler, false);
+    verifyPlugins(proto.getContainerLaunchersList(), 1, testLauncher, false);
+    verifyPlugins(proto.getTaskCommunicatorsList(), 1, testComm, true);
+
+
+    // Multiple plugin set specified. All with a payload
+    taskSchedulers = createTaskScheduelrs(3, true);
+    containerLaunchers = createContainerLaunchers(3, true);
+    taskComms = createTaskCommunicators(3, true);
+
+    servicePluginsDescriptor = ServicePluginsDescriptor.create(taskSchedulers, containerLaunchers,
+        taskComms);
+    proto = DagTypeConverters.convertServicePluginDescriptoToProto(servicePluginsDescriptor);
+    assertTrue(proto.hasUberEnabled());
+    assertTrue(proto.hasContainersEnabled());
+    assertFalse(proto.getUberEnabled());
+    assertTrue(proto.getContainersEnabled());
+    verifyPlugins(proto.getTaskSchedulersList(), 3, testScheduler, true);
+    verifyPlugins(proto.getContainerLaunchersList(), 3, testLauncher, true);
+    verifyPlugins(proto.getTaskCommunicatorsList(), 3, testComm, true);
+
+    // Single plugin set specified. One with a payload. No container execution. Uber enabled.
+    taskSchedulers = createTaskScheduelrs(1, false);
+    containerLaunchers = createContainerLaunchers(1, false);
+    taskComms = createTaskCommunicators(1, true);
+
+    servicePluginsDescriptor = ServicePluginsDescriptor.create(false, true, taskSchedulers, containerLaunchers,
+        taskComms);
+    proto = DagTypeConverters.convertServicePluginDescriptoToProto(servicePluginsDescriptor);
+    assertTrue(proto.hasUberEnabled());
+    assertTrue(proto.hasContainersEnabled());
+    assertTrue(proto.getUberEnabled());
+    assertFalse(proto.getContainersEnabled());
+    verifyPlugins(proto.getTaskSchedulersList(), 1, testScheduler, false);
+    verifyPlugins(proto.getContainerLaunchersList(), 1, testLauncher, false);
+    verifyPlugins(proto.getTaskCommunicatorsList(), 1, testComm, true);
+  }
+
+  private void verifyPlugins(List<TezNamedEntityDescriptorProto> entities, int expectedCount,
+                             String baseString, boolean hasPayload) {
+    assertEquals(expectedCount, entities.size());
+    for (int i = 0; i < expectedCount; i++) {
+      assertEquals(indexedEntity(baseString, i), entities.get(i).getName());
+      TezEntityDescriptorProto subEntityProto = entities.get(i).getEntityDescriptor();
+      assertEquals(append(indexedEntity(baseString, i), classSuffix),
+          subEntityProto.getClassName());
+      assertEquals(hasPayload, subEntityProto.hasTezUserPayload());
+      if (hasPayload) {
+        UserPayload userPayload =
+            UserPayload
+                .create(subEntityProto.getTezUserPayload().getUserPayload().asReadOnlyByteBuffer(),
+                    subEntityProto.getTezUserPayload().getVersion());
+        ByteBuffer bb = userPayload.getPayload();
+        assertNotNull(bb);
+        assertEquals(i, bb.getInt());
+      }
+    }
+  }
+
+  private TaskSchedulerDescriptor[] createTaskScheduelrs(int count, boolean withUserPayload) {
+    TaskSchedulerDescriptor[] descriptors = new TaskSchedulerDescriptor[count];
+    for (int i = 0; i < count; i++) {
+      descriptors[i] = TaskSchedulerDescriptor.create(indexedEntity(testScheduler, i),
+          append(indexedEntity(testScheduler, i), classSuffix));
+      if (withUserPayload) {
+        descriptors[i].setUserPayload(createPayload(i));
+      }
+    }
+    return descriptors;
+  }
+
+  private ContainerLauncherDescriptor[] createContainerLaunchers(int count,
+                                                                 boolean withUserPayload) {
+    ContainerLauncherDescriptor[] descriptors = new ContainerLauncherDescriptor[count];
+    for (int i = 0; i < count; i++) {
+      descriptors[i] = ContainerLauncherDescriptor.create(indexedEntity(testLauncher, i),
+          append(indexedEntity(testLauncher, i), classSuffix));
+      if (withUserPayload) {
+        descriptors[i].setUserPayload(createPayload(i));
+      }
+    }
+    return descriptors;
+  }
+
+  private TaskCommunicatorDescriptor[] createTaskCommunicators(int count, boolean withUserPayload) {
+    TaskCommunicatorDescriptor[] descriptors = new TaskCommunicatorDescriptor[count];
+    for (int i = 0; i < count; i++) {
+      descriptors[i] = TaskCommunicatorDescriptor.create(indexedEntity(testComm, i),
+          append(indexedEntity(testComm, i), classSuffix));
+      if (withUserPayload) {
+        descriptors[i].setUserPayload(createPayload(i));
+      }
+    }
+    return descriptors;
+  }
+
+  private static UserPayload createPayload(int i) {
+    ByteBuffer bb = ByteBuffer.allocate(4);
+    bb.putInt(0, i);
+    UserPayload payload = UserPayload.create(bb);
+    return payload;
+  }
+
+  private String indexedEntity(String name, int index) {
+    return name + index;
+  }
+
+  private String append(String s1, String s2) {
+    return s1 + s2;
+  }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/a169d8ae/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 011a234..9ed14d7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -408,9 +408,9 @@ public class DAGAppMaster extends AbstractService {
     }
 
     taskSchedulerDescriptors = parsePlugin(taskSchedulers,
-        (amPluginDescriptorProto == null || amPluginDescriptorProto.getTaskScedulersCount() == 0 ?
+        (amPluginDescriptorProto == null || amPluginDescriptorProto.getTaskSchedulersCount() == 0 ?
             null :
-            amPluginDescriptorProto.getTaskScedulersList()),
+            amPluginDescriptorProto.getTaskSchedulersList()),
         tezYarnEnabled, uberEnabled);
 
     containerLauncherDescriptors = parsePlugin(containerLaunchers,