You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/02/01 18:58:00 UTC

[01/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Repository: hive
Updated Branches:
  refs/heads/branch-2.0 29a988fd9 -> 98d516770


http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
new file mode 100644
index 0000000..36d8ffd
--- /dev/null
+++ b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
@@ -0,0 +1,684 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl;
+import org.apache.hadoop.hive.llap.testhelpers.ControlledClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestLlapTaskSchedulerService {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestLlapTaskSchedulerService.class);
+
+  private static final String HOST1 = "host1";
+  private static final String HOST2 = "host2";
+  private static final String HOST3 = "host3";
+
+  @Test (timeout = 5000)
+  public void testSimpleLocalAllocation() throws IOException, InterruptedException {
+
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
+
+    try {
+      Priority priority1 = Priority.newInstance(1);
+      String[] hosts1 = new String[]{HOST1};
+
+      Object task1 = new Object();
+      Object clientCookie1 = new Object();
+
+      tsWrapper.controlScheduler(true);
+      tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
+
+      tsWrapper.signalSchedulerRun();
+      tsWrapper.awaitSchedulerRun();
+
+      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
+      // TODO Verify this is on host1.
+      assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+  @Test (timeout = 5000)
+  public void testSimpleNoLocalityAllocation() throws IOException, InterruptedException {
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
+
+    try {
+      Priority priority1 = Priority.newInstance(1);
+
+      Object task1 = new Object();
+      Object clientCookie1 = new Object();
+      tsWrapper.controlScheduler(true);
+      tsWrapper.allocateTask(task1, null, priority1, clientCookie1);
+      tsWrapper.signalSchedulerRun();
+      tsWrapper.awaitSchedulerRun();
+      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
+      assertEquals(1, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+
+  @Test(timeout=5000)
+  public void testPreemption() throws InterruptedException, IOException {
+
+    Priority priority1 = Priority.newInstance(1);
+    Priority priority2 = Priority.newInstance(2);
+    String [] hosts = new String[] {HOST1};
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1);
+    try {
+
+      Object task1 = "task1";
+      Object clientCookie1 = "cookie1";
+      Object task2 = "task2";
+      Object clientCookie2 = "cookie2";
+      Object task3 = "task3";
+      Object clientCookie3 = "cookie3";
+      Object task4 = "task4";
+      Object clientCookie4 = "cookie4";
+
+      tsWrapper.controlScheduler(true);
+      tsWrapper.allocateTask(task1, hosts, priority2, clientCookie1);
+      tsWrapper.allocateTask(task2, hosts, priority2, clientCookie2);
+      tsWrapper.allocateTask(task3, hosts, priority2, clientCookie3);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(any(Object.class),
+          any(Object.class), any(Container.class));
+      assertEquals(2, tsWrapper.ts.dagStats.numLocalAllocations);
+      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+
+      reset(tsWrapper.mockAppCallback);
+
+      tsWrapper.allocateTask(task4, hosts, priority1, clientCookie4);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
+
+
+      tsWrapper.deallocateTask(task2, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
+
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
+          eq(clientCookie4), any(Container.class));
+
+    } finally {
+      tsWrapper.shutdown();
+    }
+
+  }
+
+  @Test(timeout=5000)
+  public void testNodeDisabled() throws IOException, InterruptedException {
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(10000l);
+    try {
+      Priority priority1 = Priority.newInstance(1);
+      String[] hosts1 = new String[]{HOST1};
+      Object task1 = new Object();
+      Object clientCookie1 = new Object();
+      tsWrapper.controlScheduler(true);
+      tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 1) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1),
+          any(Container.class));
+      assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
+      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+      assertEquals(0, tsWrapper.ts.dagStats.numNonLocalAllocations);
+      assertEquals(1, tsWrapper.ts.dagStats.numTotalAllocations);
+
+      tsWrapper.resetAppCallback();
+
+      tsWrapper.clock.setTime(10000l);
+      tsWrapper.rejectExecution(task1);
+
+      // Verify that the node is blacklisted
+      assertEquals(1, tsWrapper.ts.dagStats.numRejectedTasks);
+      assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
+      LlapTaskSchedulerService.NodeInfo disabledNodeInfo = tsWrapper.ts.disabledNodesQueue.peek();
+      assertNotNull(disabledNodeInfo);
+      assertEquals(HOST1, disabledNodeInfo.serviceInstance.getHost());
+      assertEquals((10000l), disabledNodeInfo.getDelay(TimeUnit.MILLISECONDS));
+      assertEquals((10000l + 10000l), disabledNodeInfo.expireTimeMillis);
+
+      Object task2 = new Object();
+      Object clientCookie2 = new Object();
+      tsWrapper.allocateTask(task2, hosts1, priority1, clientCookie2);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task2), eq(clientCookie2), any(Container.class));
+      assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
+      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+      assertEquals(1, tsWrapper.ts.dagStats.numNonLocalAllocations);
+      assertEquals(2, tsWrapper.ts.dagStats.numTotalAllocations);
+
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+  @Test(timeout=5000)
+  public void testNodeReEnabled() throws InterruptedException, IOException {
+    // Based on actual timing.
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(1000l);
+    try {
+      Priority priority1 = Priority.newInstance(1);
+      String[] hosts1 = new String[]{HOST1};
+      String[] hosts2 = new String[]{HOST2};
+      String[] hosts3 = new String[]{HOST3};
+
+      Object task1 = new Object();
+      Object clientCookie1 = new Object();
+      Object task2 = new Object();
+      Object clientCookie2 = new Object();
+      Object task3 = new Object();
+      Object clientCookie3 = new Object();
+
+      tsWrapper.controlScheduler(true);
+      tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
+      tsWrapper.allocateTask(task2, hosts2, priority1, clientCookie2);
+      tsWrapper.allocateTask(task3, hosts3, priority1, clientCookie3);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), any(Container.class));
+      assertEquals(3, tsWrapper.ts.dagStats.numLocalAllocations);
+      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+      assertEquals(3, tsWrapper.ts.dagStats.numTotalAllocations);
+
+      tsWrapper.resetAppCallback();
+
+      tsWrapper.rejectExecution(task1);
+      tsWrapper.rejectExecution(task2);
+      tsWrapper.rejectExecution(task3);
+
+      // Verify that the node is blacklisted
+      assertEquals(3, tsWrapper.ts.dagStats.numRejectedTasks);
+      assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
+      assertEquals(3, tsWrapper.ts.disabledNodesQueue.size());
+
+
+      Object task4 = new Object();
+      Object clientCookie4 = new Object();
+      Object task5 = new Object();
+      Object clientCookie5 = new Object();
+      Object task6 = new Object();
+      Object clientCookie6 = new Object();
+      tsWrapper.allocateTask(task4, hosts1, priority1, clientCookie4);
+      tsWrapper.allocateTask(task5, hosts2, priority1, clientCookie5);
+      tsWrapper.allocateTask(task6, hosts3, priority1, clientCookie6);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 6) {
+          break;
+        }
+      }
+
+      ArgumentCaptor<Container> argumentCaptor = ArgumentCaptor.forClass(Container.class);
+      verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), argumentCaptor.capture());
+
+      // which affects the locality matching
+      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+      assertEquals(6, tsWrapper.ts.dagStats.numTotalAllocations);
+
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+  @Test (timeout = 5000)
+  public void testForceLocalityTest1() throws IOException, InterruptedException {
+    // 2 hosts. 2 per host. 5 requests at the same priority.
+    // First 3 on host1, Next at host2, Last with no host.
+    // Third request on host1 should not be allocated immediately.
+    forceLocalityTest1(true);
+
+  }
+
+  @Test (timeout = 5000)
+  public void testNoForceLocalityCounterTest1() throws IOException, InterruptedException {
+    // 2 hosts. 2 per host. 5 requests at the same priority.
+    // First 3 on host1, Next at host2, Last with no host.
+    // Third should allocate on host2, 4th on host2, 5th will wait.
+
+    forceLocalityTest1(false);
+  }
+
+  private void forceLocalityTest1(boolean forceLocality) throws IOException, InterruptedException {
+    Priority priority1 = Priority.newInstance(1);
+
+    String[] hosts = new String[] {HOST1, HOST2};
+
+    String[] hostsH1 = new String[] {HOST1};
+    String[] hostsH2 = new String[] {HOST2};
+
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, (forceLocality ? -1l : 0l));
+
+    try {
+      Object task1 = "task1";
+      Object clientCookie1 = "cookie1";
+      Object task2 = "task2";
+      Object clientCookie2 = "cookie2";
+      Object task3 = "task3";
+      Object clientCookie3 = "cookie3";
+      Object task4 = "task4";
+      Object clientCookie4 = "cookie4";
+      Object task5 = "task5";
+      Object clientCookie5 = "cookie5";
+
+      tsWrapper.controlScheduler(true);
+      //H1 - should allocate
+      tsWrapper.allocateTask(task1, hostsH1, priority1, clientCookie1);
+      //H1 - should allocate
+      tsWrapper.allocateTask(task2, hostsH1, priority1, clientCookie2);
+      //H1 - no capacity if force, should allocate otherwise
+      tsWrapper.allocateTask(task3, hostsH1, priority1, clientCookie3);
+      //H2 - should allocate
+      tsWrapper.allocateTask(task4, hostsH2, priority1, clientCookie4);
+      //No location - should allocate if force, no capacity otherwise
+      tsWrapper.allocateTask(task5, null, priority1, clientCookie5);
+
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 4) {
+          break;
+        }
+      }
+
+      // Verify no preemption requests - since everything is at the same priority
+      verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
+      ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
+      verify(tsWrapper.mockAppCallback, times(4)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+      assertEquals(4, argumentCaptor.getAllValues().size());
+      assertEquals(task1, argumentCaptor.getAllValues().get(0));
+      assertEquals(task2, argumentCaptor.getAllValues().get(1));
+      if (forceLocality) {
+        // task3 not allocated
+        assertEquals(task4, argumentCaptor.getAllValues().get(2));
+        assertEquals(task5, argumentCaptor.getAllValues().get(3));
+      } else {
+        assertEquals(task3, argumentCaptor.getAllValues().get(2));
+        assertEquals(task4, argumentCaptor.getAllValues().get(3));
+      }
+
+      //Complete one task on host1.
+      tsWrapper.deallocateTask(task1, true, null);
+
+      reset(tsWrapper.mockAppCallback);
+
+      // Try scheduling again.
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 5) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
+      argumentCaptor = ArgumentCaptor.forClass(Object.class);
+      verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+      assertEquals(1, argumentCaptor.getAllValues().size());
+      if (forceLocality) {
+        assertEquals(task3, argumentCaptor.getAllValues().get(0));
+      } else {
+        assertEquals(task5, argumentCaptor.getAllValues().get(0));
+      }
+
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+  @Test(timeout = 5000)
+  public void testForcedLocalityUnknownHost() throws IOException, InterruptedException {
+    Priority priority1 = Priority.newInstance(1);
+
+    String[] hostsKnown = new String[]{HOST1};
+    String[] hostsUnknown = new String[]{HOST2};
+
+    TestTaskSchedulerServiceWrapper tsWrapper =
+        new TestTaskSchedulerServiceWrapper(2000, hostsKnown, 1, 1, -1l);
+    try {
+      Object task1 = "task1";
+      Object clientCookie1 = "cookie1";
+
+      Object task2 = "task2";
+      Object clientCookie2 = "cookie2";
+
+      tsWrapper.controlScheduler(true);
+      // Should allocate since H2 is not known.
+      tsWrapper.allocateTask(task1, hostsUnknown, priority1, clientCookie1);
+      tsWrapper.allocateTask(task2, hostsKnown, priority1, clientCookie2);
+
+
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
+          break;
+        }
+      }
+
+      ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
+      verify(tsWrapper.mockAppCallback, times(2))
+          .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+      assertEquals(2, argumentCaptor.getAllValues().size());
+      assertEquals(task1, argumentCaptor.getAllValues().get(0));
+      assertEquals(task2, argumentCaptor.getAllValues().get(1));
+
+
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+
+  @Test(timeout = 5000)
+  public void testForcedLocalityPreemption() throws IOException, InterruptedException {
+    Priority priority1 = Priority.newInstance(1);
+    Priority priority2 = Priority.newInstance(2);
+    String [] hosts = new String[] {HOST1, HOST2};
+
+    String [] hostsH1 = new String[] {HOST1};
+    String [] hostsH2 = new String[] {HOST2};
+
+    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, -1l);
+
+    // Fill up host1 with p2 tasks.
+    // Leave host2 empty
+    // Try running p1 task on host1 - should preempt
+
+    try {
+      Object task1 = "task1";
+      Object clientCookie1 = "cookie1";
+      Object task2 = "task2";
+      Object clientCookie2 = "cookie2";
+      Object task3 = "task3";
+      Object clientCookie3 = "cookie3";
+      Object task4 = "task4";
+      Object clientCookie4 = "cookie4";
+
+      tsWrapper.controlScheduler(true);
+      tsWrapper.allocateTask(task1, hostsH1, priority2, clientCookie1);
+      tsWrapper.allocateTask(task2, hostsH1, priority2, clientCookie2);
+      // This request at a lower priority should not affect anything.
+      tsWrapper.allocateTask(task3, hostsH1, priority2, clientCookie3);
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
+          break;
+        }
+      }
+
+      verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
+      ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
+      verify(tsWrapper.mockAppCallback, times(2))
+          .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+      assertEquals(2, argumentCaptor.getAllValues().size());
+      assertEquals(task1, argumentCaptor.getAllValues().get(0));
+      assertEquals(task2, argumentCaptor.getAllValues().get(1));
+
+      reset(tsWrapper.mockAppCallback);
+      // Allocate t4 at higher priority. t3 should not be allocated,
+      // and a preemption should be attempted on host1, despite host2 having available capacity
+      tsWrapper.allocateTask(task4, hostsH1, priority1, clientCookie4);
+
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
+
+      tsWrapper.deallocateTask(task1, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
+
+      while (true) {
+        tsWrapper.signalSchedulerRun();
+        tsWrapper.awaitSchedulerRun();
+        if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
+          break;
+        }
+      }
+      verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
+          eq(clientCookie4), any(Container.class));
+
+    } finally {
+      tsWrapper.shutdown();
+    }
+  }
+
+  private static class TestTaskSchedulerServiceWrapper {
+    static final Resource resource = Resource.newInstance(1024, 1);
+    Configuration conf;
+    TaskSchedulerContext mockAppCallback = mock(TaskSchedulerContext.class);
+    ControlledClock clock = new ControlledClock(new SystemClock());
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1000, 1), 1);
+    LlapTaskSchedulerServiceForTest ts;
+
+    TestTaskSchedulerServiceWrapper() throws IOException, InterruptedException {
+      this(2000l);
+    }
+
+    TestTaskSchedulerServiceWrapper(long disableTimeoutMillis) throws IOException,
+        InterruptedException {
+      this(disableTimeoutMillis, new String[]{HOST1, HOST2, HOST3}, 4,
+          ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.defaultIntVal);
+    }
+
+    TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize) throws
+        IOException, InterruptedException {
+      this(disableTimeoutMillis, hosts, numExecutors, waitQueueSize, 0l);
+    }
+
+    TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize, long localityDelayMs) throws
+        IOException, InterruptedException {
+      conf = new Configuration();
+      conf.setStrings(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, hosts);
+      conf.setInt(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, numExecutors);
+      conf.setInt(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname, waitQueueSize);
+      conf.set(ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS.varname,
+          disableTimeoutMillis + "ms");
+      conf.setBoolean(LlapFixedRegistryImpl.FIXED_REGISTRY_RESOLVE_HOST_NAMES, false);
+      conf.setLong(ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY.varname, localityDelayMs);
+
+      doReturn(appAttemptId).when(mockAppCallback).getApplicationAttemptId();
+      doReturn(11111l).when(mockAppCallback).getCustomClusterIdentifier();
+      UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+      doReturn(userPayload).when(mockAppCallback).getInitialUserPayload();
+
+      ts = new LlapTaskSchedulerServiceForTest(mockAppCallback, clock);
+
+      controlScheduler(true);
+      ts.initialize();
+      ts.start();
+      // One scheduler pass from the nodes that are added at startup
+      signalSchedulerRun();
+      controlScheduler(false);
+      awaitSchedulerRun();
+    }
+
+    void controlScheduler(boolean val) {
+      ts.forTestsetControlScheduling(val);
+    }
+
+    void signalSchedulerRun() throws InterruptedException {
+      ts.forTestSignalSchedulingRun();
+    }
+
+    void awaitSchedulerRun() throws InterruptedException {
+      ts.forTestAwaitSchedulingRun();
+    }
+    void resetAppCallback() {
+      reset(mockAppCallback);
+    }
+
+    void shutdown() {
+      ts.shutdown();
+    }
+
+    void allocateTask(Object task, String[] hosts, Priority priority, Object clientCookie) {
+      ts.allocateTask(task, resource, hosts, null, priority, null, clientCookie);
+    }
+
+    void deallocateTask(Object task, boolean succeeded, TaskAttemptEndReason endReason) {
+      ts.deallocateTask(task, succeeded, endReason, null);
+    }
+
+    void rejectExecution(Object task) {
+      ts.deallocateTask(task, false, TaskAttemptEndReason.EXECUTOR_BUSY, null);
+    }
+  }
+
+  private static class LlapTaskSchedulerServiceForTest extends LlapTaskSchedulerService {
+
+    private AtomicBoolean controlScheduling = new AtomicBoolean(false);
+    private final Lock testLock = new ReentrantLock();
+    private final Condition schedulingCompleteCondition = testLock.newCondition();
+    private boolean schedulingComplete = false;
+    private final Condition triggerSchedulingCondition = testLock.newCondition();
+    private boolean schedulingTriggered = false;
+    private final AtomicInteger numSchedulerRuns = new AtomicInteger(0);
+
+
+    public LlapTaskSchedulerServiceForTest(
+        TaskSchedulerContext appClient, Clock clock) {
+      super(appClient, clock);
+    }
+
+    @Override
+    protected void schedulePendingTasks() {
+      testLock.lock();
+      try {
+        if (controlScheduling.get()) {
+          while (!schedulingTriggered) {
+            try {
+              triggerSchedulingCondition.await();
+            } catch (InterruptedException e) {
+              throw new RuntimeException(e);
+            }
+          }
+        }
+        numSchedulerRuns.incrementAndGet();
+        super.schedulePendingTasks();
+        schedulingTriggered = false;
+        schedulingComplete = true;
+        schedulingCompleteCondition.signal();
+      } finally {
+        testLock.unlock();
+      }
+    }
+
+    // Enable or disable test scheduling control.
+    void forTestsetControlScheduling(boolean control) {
+      this.controlScheduling.set(control);
+    }
+
+    void forTestSignalSchedulingRun() throws InterruptedException {
+      testLock.lock();
+      try {
+        schedulingTriggered = true;
+        triggerSchedulingCondition.signal();
+      } finally {
+        testLock.unlock();
+      }
+    }
+
+    void forTestAwaitSchedulingRun() throws InterruptedException {
+      testLock.lock();
+      try {
+        while (!schedulingComplete) {
+          schedulingCompleteCondition.await();
+        }
+        schedulingComplete = false;
+      } finally {
+        testLock.unlock();
+      }
+    }
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d89d64c..3d8b332 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,8 +48,10 @@
     <module>ql</module>
     <module>serde</module>
     <module>service</module>
-    <module>llap-server</module>
+    <module>llap-common</module>
     <module>llap-client</module>
+    <module>llap-tez</module>
+    <module>llap-server</module>
     <module>shims</module>
     <module>spark-client</module>
     <module>storage-api</module>

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index 0745ff3..6b596d5 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -68,6 +68,11 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
+      <artifactId>hive-llap-tez</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
       <artifactId>hive-shims</artifactId>
       <version>${project.version}</version>
     </dependency>
@@ -601,6 +606,55 @@
       </exclusions>
     </dependency>
     <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
       <version>${spark.version}</version>

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 0bc6e2a..6a7d035 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.ql.exec.tez;
 
 import java.util.Collection;
 
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -44,7 +42,6 @@ import java.util.concurrent.TimeoutException;
 
 import javax.security.auth.login.LoginException;
 
-import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.hadoop.fs.FileStatus;
@@ -53,9 +50,14 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.impl.LlapProtocolClientImpl;
 import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
 import org.apache.hadoop.hive.llap.security.LlapTokenProvider;
+import org.apache.hadoop.hive.llap.tez.LlapProtocolClientProxy;
+import org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -88,9 +90,9 @@ public class TezSessionState {
   private static final Logger LOG = LoggerFactory.getLogger(TezSessionState.class.getName());
   private static final String TEZ_DIR = "_tez_session_dir";
   public static final String LLAP_SERVICE = "LLAP";
-  private static final String LLAP_SCHEDULER = "org.apache.tez.dag.app.rm.LlapTaskSchedulerService";
-  private static final String LLAP_LAUNCHER = "org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher";
-  private static final String LLAP_TASK_COMMUNICATOR = "org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator";
+  private static final String LLAP_SCHEDULER = LlapTaskSchedulerService.class.getName();
+  private static final String LLAP_LAUNCHER = LlapContainerLauncher.class.getName();
+  private static final String LLAP_TASK_COMMUNICATOR = LlapTaskCommunicator.class.getName();
 
   private HiveConf conf;
   private Path tezScratchDir;
@@ -251,27 +253,11 @@ public class TezSessionState {
       // add configs for llap-daemon-site.xml + localize llap jars
       // they cannot be referred to directly as it would be a circular depedency
       conf.addResource("llap-daemon-site.xml");
-      try {
-        final File daemonJar =
-            new File(Utilities.jarFinderGetJar(Class
-                .forName("org.apache.hadoop.hive.llap.io.api.impl.LlapInputFormat")));
-        final LocalResource daemonLr =
-            createJarLocalResource(daemonJar.toURI().toURL().toExternalForm());
-        commonLocalResources.put(utils.getBaseName(daemonLr), daemonLr);
-      } catch (ClassNotFoundException ce) {
-        throw new IOException("Cannot find LlapInputFormat in the classpath", ce);
-      }
 
-      try {
-        final File registryJar =
-            new File(Utilities.jarFinderGetJar(Class
-                .forName("org.apache.hadoop.registry.client.api.RegistryOperations")));
-        final LocalResource registryLr =
-            createJarLocalResource(registryJar.toURI().toURL().toExternalForm());
-        commonLocalResources.put(utils.getBaseName(registryLr), registryLr);
-      } catch (ClassNotFoundException ce) {
-        throw new IOException("Cannot find Hadoop Registry in the classpath", ce);
-      }
+      addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
+      addJarLRByClass(LlapProtocolClientImpl.class, commonLocalResources);
+      addJarLRByClass(LlapProtocolClientProxy.class, commonLocalResources);
+      addJarLRByClassName("org.apache.hadoop.registry.client.api.RegistryOperations", commonLocalResources);
     }
 
     // Create environment for AM.
@@ -587,6 +573,26 @@ public class TezSessionState {
     return utils.localizeResource(localFile, destFile, LocalResourceType.FILE, conf);
   }
 
+  private void addJarLRByClassName(String className, final Map<String, LocalResource> lrMap) throws
+      IOException, LoginException {
+    Class<?> clazz;
+    try {
+      clazz = Class.forName(className);
+    } catch (ClassNotFoundException e) {
+      throw new IOException("Cannot find " + className + " in classpath", e);
+    }
+    addJarLRByClass(clazz, lrMap);
+  }
+
+  private void addJarLRByClass(Class<?> clazz, final Map<String, LocalResource> lrMap) throws IOException,
+      LoginException {
+    final File jar =
+        new File(Utilities.jarFinderGetJar(clazz));
+    final LocalResource jarLr =
+        createJarLocalResource(jar.toURI().toURL().toExternalForm());
+    lrMap.put(utils.getBaseName(jarLr), jarLr);
+  }
+
   private String getSha(Path localFile) throws IOException, IllegalArgumentException {
     InputStream is = null;
     try {


[08/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
new file mode 100644
index 0000000..515bf3c
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+public class LlapNodeId {
+
+  private static final LoadingCache<LlapNodeId, LlapNodeId> CACHE =
+      CacheBuilder.newBuilder().softValues().build(
+          new CacheLoader<LlapNodeId, LlapNodeId>() {
+            @Override
+            public LlapNodeId load(LlapNodeId key) throws Exception {
+              return key;
+            }
+          });
+
+  public static LlapNodeId getInstance(String hostname, int port) {
+    return CACHE.getUnchecked(new LlapNodeId(hostname, port));
+  }
+
+
+  private final String hostname;
+  private final int port;
+
+
+  private LlapNodeId(String hostname, int port) {
+    this.hostname = hostname;
+    this.port = port;
+  }
+
+  public String getHostname() {
+    return hostname;
+  }
+
+  public int getPort() {
+    return port;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    LlapNodeId that = (LlapNodeId) o;
+
+    if (port != that.port) {
+      return false;
+    }
+    if (!hostname.equals(that.hostname)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = hostname.hashCode();
+    result = 1009 * result + port;
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return hostname + ":" + port;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java
new file mode 100644
index 0000000..cd11bdb
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.impl;
+
+import javax.annotation.Nullable;
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
+import org.apache.hadoop.security.UserGroupInformation;
+
+public class LlapManagementProtocolClientImpl implements LlapManagementProtocolPB {
+
+  private final Configuration conf;
+  private final InetSocketAddress serverAddr;
+  private final RetryPolicy retryPolicy;
+  private final SocketFactory socketFactory;
+  LlapManagementProtocolPB proxy;
+
+
+  public LlapManagementProtocolClientImpl(Configuration conf, String hostname, int port,
+                                      @Nullable RetryPolicy retryPolicy,
+                                      @Nullable SocketFactory socketFactory) {
+    this.conf = conf;
+    this.serverAddr = NetUtils.createSocketAddr(hostname, port);
+    this.retryPolicy = retryPolicy;
+    if (socketFactory == null) {
+      this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+    } else {
+      this.socketFactory = socketFactory;
+    }
+  }
+
+  public LlapManagementProtocolPB getProxy() throws IOException {
+    if (proxy == null) {
+      proxy = createProxy();
+    }
+    return proxy;
+  }
+
+  public LlapManagementProtocolPB createProxy() throws IOException {
+    RPC.setProtocolEngine(conf, LlapManagementProtocolPB.class, ProtobufRpcEngine.class);
+    ProtocolProxy<LlapManagementProtocolPB> proxy =
+        RPC.getProtocolProxy(LlapManagementProtocolPB.class, 0, serverAddr,
+            UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
+            retryPolicy);
+    return proxy.getProxy();
+  }
+
+  @Override
+  public GetTokenResponseProto getDelegationToken(RpcController controller,
+      GetTokenRequestProto request) throws ServiceException {
+    try {
+      return getProxy().getDelegationToken(null, request);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java
new file mode 100644
index 0000000..9234b7d
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.impl;
+
+import javax.annotation.Nullable;
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.security.UserGroupInformation;
+
+// TODO Change all this to be based on a regular interface instead of relying on the Proto service - Exception signatures cannot be controlled without this for the moment.
+
+
+public class LlapProtocolClientImpl implements LlapProtocolBlockingPB {
+
+  private final Configuration conf;
+  private final InetSocketAddress serverAddr;
+  private final RetryPolicy retryPolicy;
+  private final SocketFactory socketFactory;
+  LlapProtocolBlockingPB proxy;
+
+
+  public LlapProtocolClientImpl(Configuration conf, String hostname, int port,
+                                @Nullable RetryPolicy retryPolicy,
+                                @Nullable SocketFactory socketFactory) {
+    this.conf = conf;
+    this.serverAddr = NetUtils.createSocketAddr(hostname, port);
+    this.retryPolicy = retryPolicy;
+    if (socketFactory == null) {
+      this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+    } else {
+      this.socketFactory = socketFactory;
+    }
+  }
+
+  @Override
+  public SubmitWorkResponseProto submitWork(RpcController controller,
+                                                                     SubmitWorkRequestProto request) throws
+      ServiceException {
+    try {
+      return getProxy().submitWork(null, request);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
+                                                            SourceStateUpdatedRequestProto request) throws
+      ServiceException {
+    try {
+      return getProxy().sourceStateUpdated(null, request);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public QueryCompleteResponseProto queryComplete(RpcController controller,
+                                                  QueryCompleteRequestProto request) throws
+      ServiceException {
+    try {
+      return getProxy().queryComplete(null, request);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public TerminateFragmentResponseProto terminateFragment(
+      RpcController controller,
+      TerminateFragmentRequestProto request) throws ServiceException {
+    try {
+      return getProxy().terminateFragment(null, request);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  public LlapProtocolBlockingPB getProxy() throws IOException {
+    if (proxy == null) {
+      proxy = createProxy();
+    }
+    return proxy;
+  }
+
+  public LlapProtocolBlockingPB createProxy() throws IOException {
+    RPC.setProtocolEngine(conf, LlapProtocolBlockingPB.class, ProtobufRpcEngine.class);
+    ProtocolProxy<LlapProtocolBlockingPB> proxy =
+        RPC.getProtocolProxy(LlapProtocolBlockingPB.class, 0, serverAddr,
+            UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
+            retryPolicy);
+    return proxy.getProxy();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java
new file mode 100644
index 0000000..ff215d4
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.llap.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+
+@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB", protocolVersion = 1)
+@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
+@InterfaceAudience.Private
+public interface LlapManagementProtocolPB extends LlapDaemonProtocolProtos.LlapManagementProtocol.BlockingInterface {
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java
new file mode 100644
index 0000000..2bd56ca
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.security.LlapTokenSelector;
+
+@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB", protocolVersion = 1)
+@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
+@TokenInfo(LlapTokenSelector.class)
+@InterfaceAudience.Private
+public interface LlapProtocolBlockingPB extends LlapDaemonProtocolProtos.LlapDaemonProtocol.BlockingInterface {
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
new file mode 100644
index 0000000..9549567
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+import org.apache.tez.runtime.common.security.JobTokenSelector;
+
+@TokenInfo(JobTokenSelector.class)
+public interface LlapTaskUmbilicalProtocol extends VersionedProtocol {
+
+  public static final long versionID = 1L;
+
+  // From Tez. Eventually changes over to the LLAP protocol and ProtocolBuffers
+  boolean canCommit(TezTaskAttemptID taskid) throws IOException;
+  public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
+      throws IOException, TezException;
+
+  public void nodeHeartbeat(Text hostname, int port) throws IOException;
+
+  public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
new file mode 100644
index 0000000..f0bb495
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.security;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/** For now, a LLAP token gives access to any LLAP server. */
+public class LlapTokenIdentifier extends AbstractDelegationTokenIdentifier {
+  private static final String KIND = "LLAP_TOKEN";
+  public static final Text KIND_NAME = new Text(KIND);
+
+  public LlapTokenIdentifier() {
+    super();
+  }
+
+  public LlapTokenIdentifier(Text owner, Text renewer, Text realUser) {
+    super(owner, renewer, realUser);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    super.write(out);
+    // Nothing right now.
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    super.readFields(in);
+    // Nothing right now.
+  }
+
+  @Override
+  public Text getKind() {
+    return KIND_NAME;
+  }
+
+  @Override
+  public int hashCode() {
+    return -1;
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    return (other != null) && other.getClass().isAssignableFrom(this.getClass());
+  }
+
+  @Override
+  public String toString() {
+    return KIND;
+  }
+
+  @InterfaceAudience.Private
+  public static class Renewer extends Token.TrivialRenewer {
+    @Override
+    protected Text getKind() {
+      return KIND_NAME;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
new file mode 100644
index 0000000..2e99a28
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.security;
+
+import java.io.IOException;
+
+import org.apache.hadoop.security.token.Token;
+
+public interface LlapTokenProvider {
+  Token<LlapTokenIdentifier> getDelegationToken() throws IOException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
new file mode 100644
index 0000000..b6e7499
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.security;
+
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+
+public class LlapTokenSelector implements TokenSelector<LlapTokenIdentifier> {
+  private static final Log LOG = LogFactory.getLog(LlapTokenSelector.class);
+
+  @Override
+  public Token<LlapTokenIdentifier> selectToken(Text service,
+      Collection<Token<? extends TokenIdentifier>> tokens) {
+    if (service == null) return null;
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Looking for a token with service " + service);
+    }
+    for (Token<? extends TokenIdentifier> token : tokens) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Token = " + token.getKind() + "; service = " + token.getService());
+      }
+      if (LlapTokenIdentifier.KIND_NAME.equals(token.getKind())
+          && service.equals(token.getService())) {
+        @SuppressWarnings("unchecked")
+        Token<LlapTokenIdentifier> result = (Token<LlapTokenIdentifier>)token;
+        return result;
+      }
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
new file mode 100644
index 0000000..a5c3631
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GroupInputSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto;
+import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.dag.api.EntityDescriptor;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.GroupInputSpec;
+import org.apache.tez.runtime.api.impl.InputSpec;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+
+public class Converters {
+
+  public static TaskSpec getTaskSpecfromProto(FragmentSpecProto FragmentSpecProto) {
+    TezTaskAttemptID taskAttemptID =
+        TezTaskAttemptID.fromString(FragmentSpecProto.getFragmentIdentifierString());
+
+    ProcessorDescriptor processorDescriptor = null;
+    if (FragmentSpecProto.hasProcessorDescriptor()) {
+      processorDescriptor = convertProcessorDescriptorFromProto(
+          FragmentSpecProto.getProcessorDescriptor());
+    }
+
+    List<InputSpec> inputSpecList = new ArrayList<InputSpec>(FragmentSpecProto.getInputSpecsCount());
+    if (FragmentSpecProto.getInputSpecsCount() > 0) {
+      for (IOSpecProto inputSpecProto : FragmentSpecProto.getInputSpecsList()) {
+        inputSpecList.add(getInputSpecFromProto(inputSpecProto));
+      }
+    }
+
+    List<OutputSpec> outputSpecList =
+        new ArrayList<OutputSpec>(FragmentSpecProto.getOutputSpecsCount());
+    if (FragmentSpecProto.getOutputSpecsCount() > 0) {
+      for (IOSpecProto outputSpecProto : FragmentSpecProto.getOutputSpecsList()) {
+        outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
+      }
+    }
+
+    List<GroupInputSpec> groupInputSpecs =
+        new ArrayList<GroupInputSpec>(FragmentSpecProto.getGroupedInputSpecsCount());
+    if (FragmentSpecProto.getGroupedInputSpecsCount() > 0) {
+      for (GroupInputSpecProto groupInputSpecProto : FragmentSpecProto.getGroupedInputSpecsList()) {
+        groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
+      }
+    }
+
+    TaskSpec taskSpec =
+        new TaskSpec(taskAttemptID, FragmentSpecProto.getDagName(), FragmentSpecProto.getVertexName(),
+            FragmentSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList,
+            outputSpecList, groupInputSpecs);
+    return taskSpec;
+  }
+
+  public static FragmentSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
+    FragmentSpecProto.Builder builder = FragmentSpecProto.newBuilder();
+    builder.setFragmentIdentifierString(taskSpec.getTaskAttemptID().toString());
+    builder.setDagName(taskSpec.getDAGName());
+    builder.setVertexName(taskSpec.getVertexName());
+    builder.setVertexParallelism(taskSpec.getVertexParallelism());
+    builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId());
+    builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId());
+
+    if (taskSpec.getProcessorDescriptor() != null) {
+      builder.setProcessorDescriptor(
+          convertToProto(taskSpec.getProcessorDescriptor()));
+    }
+
+    if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
+      for (InputSpec inputSpec : taskSpec.getInputs()) {
+        builder.addInputSpecs(convertInputSpecToProto(inputSpec));
+      }
+    }
+
+    if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
+      for (OutputSpec outputSpec : taskSpec.getOutputs()) {
+        builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
+      }
+    }
+
+    if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
+      for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
+        builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
+
+      }
+    }
+    return builder.build();
+  }
+
+  private static ProcessorDescriptor convertProcessorDescriptorFromProto(
+      EntityDescriptorProto proto) {
+    String className = proto.getClassName();
+    UserPayload payload = convertPayloadFromProto(proto);
+    ProcessorDescriptor pd = ProcessorDescriptor.create(className);
+    setUserPayload(pd, payload);
+    return pd;
+  }
+
+  private static EntityDescriptorProto convertToProto(
+      EntityDescriptor<?> descriptor) {
+    EntityDescriptorProto.Builder builder = EntityDescriptorProto
+        .newBuilder();
+    builder.setClassName(descriptor.getClassName());
+
+    UserPayload userPayload = descriptor.getUserPayload();
+    if (userPayload != null) {
+      UserPayloadProto.Builder payloadBuilder = UserPayloadProto.newBuilder();
+      if (userPayload.hasPayload()) {
+        payloadBuilder.setUserPayload(ByteString.copyFrom(userPayload.getPayload()));
+        payloadBuilder.setVersion(userPayload.getVersion());
+      }
+      builder.setUserPayload(payloadBuilder.build());
+    }
+    if (descriptor.getHistoryText() != null) {
+      try {
+        builder.setHistoryText(TezCommonUtils.compressByteArrayToByteString(
+            descriptor.getHistoryText().getBytes("UTF-8")));
+      } catch (IOException e) {
+        throw new TezUncheckedException(e);
+      }
+    }
+    return builder.build();
+  }
+
+  private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
+    InputDescriptor inputDescriptor = null;
+    if (inputSpecProto.hasIoDescriptor()) {
+      inputDescriptor =
+          convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor());
+    }
+    InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
+        inputSpecProto.getPhysicalEdgeCount());
+    return inputSpec;
+  }
+
+  private static InputDescriptor convertInputDescriptorFromProto(
+      EntityDescriptorProto proto) {
+    String className = proto.getClassName();
+    UserPayload payload = convertPayloadFromProto(proto);
+    InputDescriptor id = InputDescriptor.create(className);
+    setUserPayload(id, payload);
+    return id;
+  }
+
+  private static OutputDescriptor convertOutputDescriptorFromProto(
+      EntityDescriptorProto proto) {
+    String className = proto.getClassName();
+    UserPayload payload = convertPayloadFromProto(proto);
+    OutputDescriptor od = OutputDescriptor.create(className);
+    setUserPayload(od, payload);
+    return od;
+  }
+
+  private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
+    IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+    if (inputSpec.getSourceVertexName() != null) {
+      builder.setConnectedVertexName(inputSpec.getSourceVertexName());
+    }
+    if (inputSpec.getInputDescriptor() != null) {
+      builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor()));
+    }
+    builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
+    return builder.build();
+  }
+
+  private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
+    OutputDescriptor outputDescriptor = null;
+    if (outputSpecProto.hasIoDescriptor()) {
+      outputDescriptor =
+          convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor());
+    }
+    OutputSpec outputSpec =
+        new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
+            outputSpecProto.getPhysicalEdgeCount());
+    return outputSpec;
+  }
+
+  public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
+    IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+    if (outputSpec.getDestinationVertexName() != null) {
+      builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
+    }
+    if (outputSpec.getOutputDescriptor() != null) {
+      builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor()));
+    }
+    builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
+    return builder.build();
+  }
+
+  private static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
+    GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
+        groupInputSpecProto.getGroupVerticesList(),
+        convertInputDescriptorFromProto(groupInputSpecProto.getMergedInputDescriptor()));
+    return groupSpec;
+  }
+
+  private static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
+    GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
+    builder.setGroupName(groupInputSpec.getGroupName());
+    builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
+    builder.setMergedInputDescriptor(convertToProto(groupInputSpec.getMergedInputDescriptor()));
+    return builder.build();
+  }
+
+
+  private static void setUserPayload(EntityDescriptor<?> entity, UserPayload payload) {
+    if (payload != null) {
+      entity.setUserPayload(payload);
+    }
+  }
+
+  private static UserPayload convertPayloadFromProto(
+      EntityDescriptorProto proto) {
+    UserPayload userPayload = null;
+    if (proto.hasUserPayload()) {
+      if (proto.getUserPayload().hasUserPayload()) {
+        userPayload =
+            UserPayload.create(proto.getUserPayload().getUserPayload().asReadOnlyByteBuffer(), proto.getUserPayload().getVersion());
+      } else {
+        userPayload = UserPayload.create(null);
+      }
+    }
+    return userPayload;
+  }
+
+  public static SourceStateProto fromVertexState(VertexState state) {
+    switch (state) {
+      case SUCCEEDED:
+        return SourceStateProto.S_SUCCEEDED;
+      case RUNNING:
+        return SourceStateProto.S_RUNNING;
+      default:
+        throw new RuntimeException("Unexpected state: " + state);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/protobuf/LlapDaemonProtocol.proto
----------------------------------------------------------------------
diff --git a/llap-common/src/protobuf/LlapDaemonProtocol.proto b/llap-common/src/protobuf/LlapDaemonProtocol.proto
new file mode 100644
index 0000000..944c96c
--- /dev/null
+++ b/llap-common/src/protobuf/LlapDaemonProtocol.proto
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hive.llap.daemon.rpc";
+option java_outer_classname = "LlapDaemonProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+// TODO Change this as the interface evolves. Currently using Tez constructs.
+
+message UserPayloadProto {
+  optional bytes user_payload = 1;
+  optional int32 version = 2;
+}
+
+message EntityDescriptorProto {
+  optional string class_name = 1;
+  optional UserPayloadProto user_payload = 2;
+  optional bytes history_text = 3;
+}
+
+message IOSpecProto {
+  optional string connected_vertex_name = 1;
+  optional EntityDescriptorProto io_descriptor = 2;
+  optional int32 physical_edge_count = 3;
+}
+
+message GroupInputSpecProto {
+  optional string group_name = 1;
+  repeated string group_vertices = 2;
+  optional EntityDescriptorProto merged_input_descriptor = 3;
+}
+
+
+message FragmentSpecProto {
+  optional string fragment_identifier_string = 1;
+  optional string dag_name = 2;
+  optional int32 dag_id = 11;
+  optional string vertex_name = 3;
+  optional EntityDescriptorProto processor_descriptor = 4;
+  repeated IOSpecProto input_specs = 5;
+  repeated IOSpecProto output_specs = 6;
+  repeated GroupInputSpecProto grouped_input_specs = 7;
+  optional int32 vertex_parallelism = 8;
+  optional int32 fragment_number =9;
+  optional int32 attempt_number = 10;
+}
+
+message FragmentRuntimeInfo {
+  optional int32 num_self_and_upstream_tasks = 1;
+  optional int32 num_self_and_upstream_completed_tasks = 2;
+  optional int32 within_dag_priority = 3;
+  optional int64 dag_start_time = 4;
+  optional int64 first_attempt_start_time = 5;
+  optional int64 current_attempt_start_time = 6;
+}
+
+enum SourceStateProto {
+  S_SUCCEEDED = 1;
+  S_RUNNING = 2;
+}
+
+message QueryIdentifierProto {
+  optional string app_identifier = 1;
+  optional int32 dag_identifier = 2;
+}
+
+message SubmitWorkRequestProto {
+  optional string container_id_string = 1;
+  optional string am_host = 2;
+  optional int32 am_port = 3;
+  optional string token_identifier = 4;
+  optional bytes credentials_binary = 5;
+  optional string user = 6;
+  optional string application_id_string = 7;
+  optional int32 app_attempt_number = 8;
+  optional FragmentSpecProto fragment_spec = 9;
+  optional FragmentRuntimeInfo fragment_runtime_info = 10;
+}
+
+enum SubmissionStateProto {
+  ACCEPTED = 1;
+  REJECTED = 2;
+  EVICTED_OTHER = 3;
+}
+
+message SubmitWorkResponseProto {
+  optional SubmissionStateProto submission_state = 1;
+}
+
+message SourceStateUpdatedRequestProto {
+  optional QueryIdentifierProto query_identifier = 1;
+  optional string src_name = 2;
+  optional SourceStateProto state = 3;
+}
+
+message SourceStateUpdatedResponseProto {
+}
+
+message QueryCompleteRequestProto {
+  optional string query_id = 1;
+  optional QueryIdentifierProto query_identifier = 2;
+  optional int64 delete_delay = 4 [default = 0];
+}
+
+message QueryCompleteResponseProto {
+}
+
+message TerminateFragmentRequestProto {
+  optional QueryIdentifierProto query_identifier = 1;
+  optional string fragment_identifier_string = 2;
+}
+
+message TerminateFragmentResponseProto {
+}
+
+message GetTokenRequestProto {
+}
+
+message GetTokenResponseProto {
+  optional bytes token = 1;
+}
+
+service LlapDaemonProtocol {
+  rpc submitWork(SubmitWorkRequestProto) returns (SubmitWorkResponseProto);
+  rpc sourceStateUpdated(SourceStateUpdatedRequestProto) returns (SourceStateUpdatedResponseProto);
+  rpc queryComplete(QueryCompleteRequestProto) returns (QueryCompleteResponseProto);
+  rpc terminateFragment(TerminateFragmentRequestProto) returns (TerminateFragmentResponseProto);
+}
+
+service LlapManagementProtocol {
+  rpc getDelegationToken(GetTokenRequestProto) returns (GetTokenResponseProto);
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java
----------------------------------------------------------------------
diff --git a/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java b/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java
new file mode 100644
index 0000000..fec1c35
--- /dev/null
+++ b/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.llap.testhelpers;
+
+import org.apache.hadoop.yarn.util.Clock;
+
+public class ControlledClock implements Clock {
+  private long time = -1;
+  private final Clock actualClock;
+  public ControlledClock(Clock actualClock) {
+    this.actualClock = actualClock;
+  }
+  public synchronized void setTime(long time) {
+    this.time = time;
+  }
+  public synchronized void reset() {
+    time = -1;
+  }
+
+  @Override
+  public synchronized long getTime() {
+    if (time != -1) {
+      return time;
+    }
+    return actualClock.getTime();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index ab25f66..4011da0 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -32,7 +32,7 @@
   </properties>
 
   <dependencies>
-    <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+    <!-- dependencies are always listed in sorted order by groupId, artifactId -->
     <!-- intra-project -->
     <dependency>
       <groupId>org.apache.hive</groupId>
@@ -46,11 +46,21 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
+      <artifactId>hive-llap-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
       <artifactId>hive-llap-client</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
+      <artifactId>hive-llap-tez</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
       <artifactId>hive-shims</artifactId>
       <version>${project.version}</version>
     </dependency>
@@ -90,77 +100,66 @@
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>
       <optional>true</optional>
-            <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
-   </dependency>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-core</artifactId>
       <version>${hadoop.version}</version>
       <optional>true</optional>
-             <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
-  </dependency>
-    <dependency>
-      <groupId>org.apache.tez</groupId>
-      <artifactId>tez-runtime-internals</artifactId>
-      <version>${tez.version}</version>
-      <optional>true</optional>
-            <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
-   </dependency>
-    <dependency>
-      <groupId>org.apache.tez</groupId>
-      <artifactId>tez-runtime-library</artifactId>
-      <version>${tez.version}</version>
-      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.tez</groupId>
-      <artifactId>tez-mapreduce</artifactId>
+      <artifactId>tez-runtime-internals</artifactId>
       <version>${tez.version}</version>
       <optional>true</optional>
-             <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.tez</groupId>
-      <artifactId>tez-dag</artifactId>
+      <artifactId>tez-runtime-library</artifactId>
       <version>${tez.version}</version>
       <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
+
     <!-- test inter-project -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -168,17 +167,17 @@
       <version>${hadoop.version}</version>
       <classifier>tests</classifier>
       <scope>test</scope>
-            <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
-   </dependency>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
@@ -225,15 +224,15 @@
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-server</artifactId>
       <version>${hbase.version}</version>
-          <exclusions>
-             <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -248,54 +247,6 @@
     </dependency>
   </dependencies>
 
-  <profiles>
-    <profile>
-      <id>protobuf</id>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-antrun-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>generate-protobuf-sources</id>
-                <phase>generate-sources</phase>
-                <configuration>
-                  <target>
-                    <property name="protobuf.src.dir"  location="${basedir}/src/protobuf"/>
-                    <property name="protobuf.build.dir"  location="${basedir}/src/gen/protobuf/gen-java"/>
-                    <echo>Building LLAP-Server Protobuf</echo>
-                    <mkdir dir="${protobuf.build.dir}"/>
-                    <exec executable="protoc" failonerror="true">
-                      <arg value="--java_out=${protobuf.build.dir}"/>
-                      <arg value="-I=${protobuf.src.dir}"/>
-                      <arg value="${protobuf.src.dir}/LlapDaemonProtocol.proto"/>
-                    </exec>
-                  </target>
-                </configuration>
-                <goals>
-                  <goal>run</goal>
-                </goals>
-              </execution>
-            </executions>
-          </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-jar-plugin</artifactId>
-            <executions>
-              <execution>
-                <goals>
-                  <goal>test-jar</goal>
-                </goals>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-  </profiles>
-
-
   <build>
     <sourceDirectory>${basedir}/src/java</sourceDirectory>
     <testSourceDirectory>${basedir}/src/test</testSourceDirectory>


[02/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
new file mode 100644
index 0000000..3bca0da
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -0,0 +1,1512 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.commons.lang.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.registry.ServiceInstance;
+import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
+import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Clock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapTaskSchedulerService extends TaskScheduler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LlapTaskSchedulerService.class);
+
+  private final Configuration conf;
+
+  // interface into the registry service
+  private ServiceInstanceSet activeInstances;
+
+  // Tracks all instances, including ones which have been disabled in the past.
+  // LinkedHashMap to provide the same iteration order when selecting a random host.
+  @VisibleForTesting
+  final Map<ServiceInstance, NodeInfo> instanceToNodeMap = new LinkedHashMap<>();
+  // TODO Ideally, remove elements from this once it's known that no tasks are linked to the instance (all deallocated)
+
+  // Tracks tasks which could not be allocated immediately.
+  @VisibleForTesting
+  final TreeMap<Priority, List<TaskInfo>> pendingTasks = new TreeMap<>(new Comparator<Priority>() {
+    @Override
+    public int compare(Priority o1, Priority o2) {
+      return o1.getPriority() - o2.getPriority();
+    }
+  });
+
+  // Tracks running and queued tasks. Cleared after a task completes.
+  private final ConcurrentMap<Object, TaskInfo> knownTasks = new ConcurrentHashMap<>();
+  private final TreeMap<Integer, TreeSet<TaskInfo>> runningTasks = new TreeMap<>();
+  private static final TaskStartComparator TASK_INFO_COMPARATOR = new TaskStartComparator();
+
+  // Queue for disabled nodes. Nodes make it out of this queue when their expiration timeout is hit.
+  @VisibleForTesting
+  final DelayQueue<NodeInfo> disabledNodesQueue = new DelayQueue<>();
+
+  private final boolean forceLocation;
+
+  private final ContainerFactory containerFactory;
+  private final Random random = new Random();
+  private final Clock clock;
+
+  private final ListeningExecutorService nodeEnabledExecutor;
+  private final NodeEnablerCallable nodeEnablerCallable =
+      new NodeEnablerCallable();
+
+  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+  private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
+  private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
+
+  private final Lock scheduleLock = new ReentrantLock();
+  private final Condition scheduleCondition = scheduleLock.newCondition();
+  private final AtomicBoolean pendingScheduleInvodations = new AtomicBoolean(false);
+  private final ListeningExecutorService schedulerExecutor;
+  private final SchedulerCallable schedulerCallable = new SchedulerCallable();
+
+  private final AtomicBoolean isStopped = new AtomicBoolean(false);
+  // Tracks total pending preemptions.
+  private final AtomicInteger pendingPreemptions = new AtomicInteger(0);
+  // Tracks pending preemptions per host, using the hostname || Always to be accessed inside a lock
+  private final Map<String, MutableInt> pendingPreemptionsPerHost = new HashMap<>();
+
+  private final NodeBlacklistConf nodeBlacklistConf;
+
+  // Per daemon
+  private final int memoryPerInstance;
+  private final int coresPerInstance;
+  private final int executorsPerInstance;
+
+  private final int numSchedulableTasksPerNode;
+
+  // Per Executor Thread
+  private final Resource resourcePerExecutor;
+
+  private final LlapRegistryService registry = new LlapRegistryService(false);
+
+  private volatile ListenableFuture<Void> nodeEnablerFuture;
+  private volatile ListenableFuture<Void> schedulerFuture;
+
+  @VisibleForTesting
+  private final AtomicInteger dagCounter = new AtomicInteger(1);
+  // Statistics to track allocations
+  // All of stats variables are visible for testing.
+  @VisibleForTesting
+  StatsPerDag dagStats = new StatsPerDag();
+
+  public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) {
+    this(taskSchedulerContext, new SystemClock());
+  }
+
+  @VisibleForTesting
+  public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock) {
+    super(taskSchedulerContext);
+    this.clock = clock;
+    try {
+      this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
+    } catch (IOException e) {
+      throw new TezUncheckedException(
+          "Failed to parse user payload for " + LlapTaskSchedulerService.class.getSimpleName(), e);
+    }
+    this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(),
+        taskSchedulerContext.getCustomClusterIdentifier());
+    this.memoryPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB);
+    this.coresPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE);
+    this.executorsPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
+    this.nodeBlacklistConf = new NodeBlacklistConf(
+        HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS,
+            TimeUnit.MILLISECONDS),
+        HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MAX_TIMEOUT_MS,
+            TimeUnit.MILLISECONDS),
+        HiveConf.getFloatVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_DISABLE_BACK_OFF_FACTOR));
+
+    this.numSchedulableTasksPerNode =
+        HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NUM_SCHEDULABLE_TASKS_PER_NODE);
+
+    long localityDelayMs = HiveConf
+        .getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY, TimeUnit.MILLISECONDS);
+    if (localityDelayMs == -1) {
+      this.forceLocation = true;
+    } else {
+      this.forceLocation = false;
+    }
+
+    int memoryPerExecutor = (int) (memoryPerInstance / (float) executorsPerInstance);
+    int coresPerExecutor = (int) (coresPerInstance / (float) executorsPerInstance);
+    this.resourcePerExecutor = Resource.newInstance(memoryPerExecutor, coresPerExecutor);
+
+    String instanceId = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
+
+    Preconditions.checkNotNull(instanceId, ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname
+        + " must be defined");
+
+    ExecutorService executorServiceRaw =
+        Executors.newFixedThreadPool(1,
+            new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapSchedulerNodeEnabler").build());
+    nodeEnabledExecutor = MoreExecutors.listeningDecorator(executorServiceRaw);
+
+    ExecutorService schedulerExecutorServiceRaw = Executors.newFixedThreadPool(1,
+        new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build());
+    schedulerExecutor = MoreExecutors.listeningDecorator(schedulerExecutorServiceRaw);
+
+    LOG.info("Running with configuration: " + "memoryPerInstance=" + memoryPerInstance
+        + ", vCoresPerInstance=" + coresPerInstance + ", executorsPerInstance="
+        + executorsPerInstance + ", resourcePerInstanceInferred=" + resourcePerExecutor
+        + ", nodeBlacklistConf=" + nodeBlacklistConf
+        + ", forceLocation=" + forceLocation);
+  }
+
+  @Override
+  public void initialize() {
+    registry.init(conf);
+  }
+
+  @Override
+  public void start() throws IOException {
+    writeLock.lock();
+    try {
+      nodeEnablerFuture = nodeEnabledExecutor.submit(nodeEnablerCallable);
+      Futures.addCallback(nodeEnablerFuture, new FutureCallback<Void>() {
+        @Override
+        public void onSuccess(Void result) {
+          LOG.info("NodeEnabledThread exited");
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+          LOG.warn("NodeEnabledThread exited with error", t);
+        }
+      });
+      schedulerFuture = schedulerExecutor.submit(schedulerCallable);
+      Futures.addCallback(schedulerFuture, new FutureCallback<Void>() {
+        @Override
+        public void onSuccess(Void result) {
+          LOG.info("SchedulerThread exited");
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+          LOG.warn("SchedulerThread exited with error", t);
+        }
+      });
+      registry.start();
+      activeInstances = registry.getInstances();
+      for (ServiceInstance inst : activeInstances.getAll().values()) {
+        addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public void shutdown() {
+    writeLock.lock();
+    try {
+      if (!this.isStopped.getAndSet(true)) {
+        nodeEnablerCallable.shutdown();
+        if (nodeEnablerFuture != null) {
+          nodeEnablerFuture.cancel(true);
+        }
+        nodeEnabledExecutor.shutdownNow();
+
+        schedulerCallable.shutdown();
+        if (schedulerFuture != null) {
+          schedulerFuture.cancel(true);
+        }
+        schedulerExecutor.shutdownNow();
+
+        if (registry != null) {
+          registry.stop();
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public Resource getTotalResources() {
+    int memory = 0;
+    int vcores = 0;
+    readLock.lock();
+    try {
+      for (ServiceInstance inst : activeInstances.getAll().values()) {
+        if (inst.isAlive()) {
+          Resource r = inst.getResource();
+          LOG.info("Found instance " + inst);
+          memory += r.getMemory();
+          vcores += r.getVirtualCores();
+        } else {
+          LOG.info("Ignoring dead instance " + inst);
+        }
+      }
+    } finally {
+      readLock.unlock();
+    }
+
+    return Resource.newInstance(memory, vcores);
+  }
+
+  /**
+   * The difference between this and getTotalResources() is that this only gives currently free
+   * resource instances, while the other lists all the instances that may become available in a
+   * while.
+   */
+  @Override
+  public Resource getAvailableResources() {
+    // need a state store eventually for current state & measure backoffs
+    int memory = 0;
+    int vcores = 0;
+    readLock.lock();
+    try {
+      for (Entry<ServiceInstance, NodeInfo> entry : instanceToNodeMap.entrySet()) {
+        if (entry.getKey().isAlive() && !entry.getValue().isDisabled()) {
+          Resource r = entry.getKey().getResource();
+          memory += r.getMemory();
+          vcores += r.getVirtualCores();
+        }
+      }
+    } finally {
+      readLock.unlock();
+    }
+
+    return Resource.newInstance(memory, vcores);
+  }
+
+  @Override
+  public int getClusterNodeCount() {
+    readLock.lock();
+    try {
+      int n = 0;
+      for (ServiceInstance inst : activeInstances.getAll().values()) {
+        if (inst.isAlive()) {
+          n++;
+        }
+      }
+      return n;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public void dagComplete() {
+    // This is effectively DAG completed, and can be used to reset statistics being tracked.
+    LOG.info("DAG: " + dagCounter.get() + " completed. Scheduling stats: " + dagStats);
+    dagCounter.incrementAndGet();
+    dagStats = new StatsPerDag();
+  }
+
+  @Override
+  public void blacklistNode(NodeId nodeId) {
+    LOG.info("BlacklistNode not supported");
+  }
+
+  @Override
+  public void unblacklistNode(NodeId nodeId) {
+    LOG.info("unBlacklistNode not supported");
+  }
+
+  @Override
+  public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
+      Priority priority, Object containerSignature, Object clientCookie) {
+    TaskInfo taskInfo =
+        new TaskInfo(task, clientCookie, priority, capability, hosts, racks, clock.getTime());
+    writeLock.lock();
+    try {
+      dagStats.registerTaskRequest(hosts, racks);
+    } finally {
+      writeLock.unlock();
+    }
+    addPendingTask(taskInfo);
+    trySchedulingPendingTasks();
+  }
+
+  @Override
+  public void allocateTask(Object task, Resource capability, ContainerId containerId,
+      Priority priority, Object containerSignature, Object clientCookie) {
+    // Container affinity can be implemented as Host affinity for LLAP. Not required until
+    // 1:1 edges are used in Hive.
+    TaskInfo taskInfo =
+        new TaskInfo(task, clientCookie, priority, capability, null, null, clock.getTime());
+    writeLock.lock();
+    try {
+      dagStats.registerTaskRequest(null, null);
+    } finally {
+      writeLock.unlock();
+    }
+    addPendingTask(taskInfo);
+    trySchedulingPendingTasks();
+  }
+
+
+  // This may be invoked before a container is ever assigned to a task. allocateTask... app decides
+  // the task is no longer required, and asks for a de-allocation.
+  @Override
+  public boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason, String diagnostics) {
+    writeLock.lock(); // Updating several local structures
+    TaskInfo taskInfo;
+    try {
+      taskInfo = unregisterTask(task);
+      if (taskInfo == null) {
+        LOG.error("Could not determine ContainerId for task: "
+            + task
+            + " . Could have hit a race condition. Ignoring."
+            + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
+        return false;
+      }
+      if (taskInfo.containerId == null) {
+        if (taskInfo.assigned) {
+          LOG.error("Task: "
+              + task
+              + " assigned, but could not find the corresponding containerId."
+              + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
+        } else {
+          LOG.info("Ignoring deallocate request for task " + task
+              + " which hasn't been assigned to a container");
+          removePendingTask(taskInfo);
+        }
+        return false;
+      }
+      ServiceInstance assignedInstance = taskInfo.assignedInstance;
+      assert assignedInstance != null;
+
+      NodeInfo nodeInfo = instanceToNodeMap.get(assignedInstance);
+      assert nodeInfo != null;
+
+      // Re-enable the node if preempted
+      if (taskInfo.preempted) {
+        LOG.info("Processing deallocateTask for {} which was preempted, EndReason={}", task, endReason);
+        unregisterPendingPreemption(taskInfo.assignedInstance.getHost());
+        nodeInfo.registerUnsuccessfulTaskEnd(true);
+        if (nodeInfo.isDisabled()) {
+          // Re-enable the node. If a task succeeded, a slot may have become available.
+          // Also reset commFailures since a task was able to communicate back and indicate success.
+          nodeInfo.enableNode();
+          // Re-insert into the queue to force the poll thread to remove the element.
+          if ( disabledNodesQueue.remove(nodeInfo)) {
+            disabledNodesQueue.add(nodeInfo);
+          }
+        }
+        // In case of success, trigger a scheduling run for pending tasks.
+        trySchedulingPendingTasks();
+      } else {
+        if (taskSucceeded) {
+          // The node may have been blacklisted at this point - which means it may not be in the
+          // activeNodeList.
+
+          nodeInfo.registerTaskSuccess();
+
+          if (nodeInfo.isDisabled()) {
+            // Re-enable the node. If a task succeeded, a slot may have become available.
+            // Also reset commFailures since a task was able to communicate back and indicate success.
+            nodeInfo.enableNode();
+            // Re-insert into the queue to force the poll thread to remove the element.
+            if (disabledNodesQueue.remove(nodeInfo)) {
+              disabledNodesQueue.add(nodeInfo);
+            }
+          }
+          // In case of success, trigger a scheduling run for pending tasks.
+          trySchedulingPendingTasks();
+
+        } else if (!taskSucceeded) {
+          nodeInfo.registerUnsuccessfulTaskEnd(false);
+          if (endReason != null && EnumSet
+              .of(TaskAttemptEndReason.EXECUTOR_BUSY, TaskAttemptEndReason.COMMUNICATION_ERROR)
+              .contains(endReason)) {
+            if (endReason == TaskAttemptEndReason.COMMUNICATION_ERROR) {
+              dagStats.registerCommFailure(taskInfo.assignedInstance.getHost());
+            } else if (endReason == TaskAttemptEndReason.EXECUTOR_BUSY) {
+              dagStats.registerTaskRejected(taskInfo.assignedInstance.getHost());
+            }
+          }
+          boolean commFailure =
+              endReason != null && endReason == TaskAttemptEndReason.COMMUNICATION_ERROR;
+          disableInstance(assignedInstance, commFailure);
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+    getContext().containerBeingReleased(taskInfo.containerId);
+    return true;
+  }
+
+  @Override
+  public Object deallocateContainer(ContainerId containerId) {
+    LOG.debug("Ignoring deallocateContainer for containerId: " + containerId);
+    // Containers are not being tracked for re-use.
+    // This is safe to ignore since a deallocate task will come in.
+    return null;
+  }
+
+  @Override
+  public void setShouldUnregister() {
+
+  }
+
+  @Override
+  public boolean hasUnregistered() {
+    // Nothing to do. No registration involved.
+    return true;
+  }
+
+  private ExecutorService createAppCallbackExecutorService() {
+    return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
+        .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
+  }
+
+  /**
+   * @param request the list of preferred hosts. null implies any host
+   * @return
+   */
+  private SelectHostResult selectHost(TaskInfo request) {
+    String[] requestedHosts = request.requestedHosts;
+    readLock.lock(); // Read-lock. Not updating any stats at the moment.
+    try {
+      // Check if any hosts are active.
+      if (getAvailableResources().getMemory() <= 0) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Refreshing instances since total memory is 0");
+        }
+        refreshInstances();
+      }
+
+      // If there's no memory available, fail
+      if (getTotalResources().getMemory() <= 0) {
+        return SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY;
+      }
+
+      if (requestedHosts != null && requestedHosts.length > 0) {
+        int prefHostCount = -1;
+        boolean requestedHostExists = false;
+        for (String host : requestedHosts) {
+          prefHostCount++;
+          // Pick the first host always. Weak attempt at cache affinity.
+          Set<ServiceInstance> instances = activeInstances.getByHost(host);
+          if (!instances.isEmpty()) {
+            requestedHostExists = true;
+            for (ServiceInstance inst : instances) {
+              NodeInfo nodeInfo = instanceToNodeMap.get(inst);
+              if (nodeInfo != null && nodeInfo.canAcceptTask()) {
+                LOG.info("Assigning " + inst + " when looking for " + host + "." +
+                    " FirstRequestedHost=" + (prefHostCount == 0) +
+                    (requestedHosts.length > 1 ? "#prefLocations=" + requestedHosts.length : ""));
+                return new SelectHostResult(inst, nodeInfo);
+              }
+            }
+          }
+        }
+        // Check if forcing the location is required.
+        if (forceLocation) {
+          if (requestedHostExists) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Skipping non-local location allocation for [" + request.task +
+                  "] when trying to allocate on [" + Arrays.toString(requestedHosts) + "]");
+            }
+            return SELECT_HOST_RESULT_DELAYED_LOCALITY;
+          } else {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Not skipping non-local location allocation for [" + request.task +
+                  "] when trying to allocate on [" + Arrays.toString(requestedHosts) +
+                  "] since none of these hosts are part of the known list");
+            }
+          }
+        }
+      }
+      /* fall through - miss in locality (random scheduling) */
+      Entry<ServiceInstance, NodeInfo>[] all =
+          instanceToNodeMap.entrySet().toArray(new Entry[instanceToNodeMap.size()]);
+      // Check again
+      if (all.length > 0) {
+        int n = random.nextInt(all.length);
+        // start at random offset and iterate whole list
+        for (int i = 0; i < all.length; i++) {
+          Entry<ServiceInstance, NodeInfo> inst = all[(i + n) % all.length];
+          if (inst.getValue().canAcceptTask()) {
+            LOG.info("Assigning " + inst + " when looking for any host, from #hosts=" + all.length +
+                ", requestedHosts=" +
+                ((requestedHosts == null || requestedHosts.length == 0) ? "null" :
+                    Arrays.toString(requestedHosts)));
+            return new SelectHostResult(inst.getKey(), inst.getValue());
+          }
+        }
+      }
+      return SELECT_HOST_RESULT_DELAYED_RESOURCES;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  // TODO Each refresh operation should addNodes if they don't already exist.
+  // Even better would be to get notifications from the service impl when a node gets added or removed.
+  // Instead of having to walk through the entire list. The computation of a node getting added or
+  // removed already exists in the DynamicRegistry implementation.
+  private void refreshInstances() {
+    try {
+      activeInstances.refresh(); // handles its own sync
+    } catch (IOException ioe) {
+      LOG.warn("Could not refresh list of active instances", ioe);
+    }
+  }
+
+  private void scanForNodeChanges() {
+    /* check again whether nodes are disabled or just missing */
+    writeLock.lock();
+    try {
+      for (ServiceInstance inst : activeInstances.getAll().values()) {
+        if (inst.isAlive() && instanceToNodeMap.containsKey(inst) == false) {
+          /* that's a good node, not added to the allocations yet */
+          LOG.info("Found a new node: " + inst + ".");
+          addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void addNode(ServiceInstance inst, NodeInfo node) {
+    LOG.info("Adding node: " + inst);
+    instanceToNodeMap.put(inst, node);
+    // Trigger scheduling since a new node became available.
+    trySchedulingPendingTasks();
+  }
+
+  private void reenableDisabledNode(NodeInfo nodeInfo) {
+    writeLock.lock();
+    try {
+      if (nodeInfo.hadCommFailure()) {
+        // If the node being re-enabled was not marked busy previously, then it was disabled due to
+        // some other failure. Refresh the service list to see if it's been removed permanently.
+        refreshInstances();
+      }
+      LOG.info("Attempting to re-enable node: " + nodeInfo.getServiceInstance());
+      if (nodeInfo.getServiceInstance().isAlive()) {
+        nodeInfo.enableNode();
+      } else {
+        if (LOG.isInfoEnabled()) {
+          LOG.info("Removing dead node " + nodeInfo);
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void disableInstance(ServiceInstance instance, boolean isCommFailure) {
+    writeLock.lock();
+    try {
+      NodeInfo nodeInfo = instanceToNodeMap.get(instance);
+      if (nodeInfo == null || nodeInfo.isDisabled()) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Node: " + instance + " already disabled, or invalid. Not doing anything.");
+        }
+      } else {
+        nodeInfo.disableNode(isCommFailure);
+        // TODO: handle task to container map events in case of hard failures
+        disabledNodesQueue.add(nodeInfo);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void addPendingTask(TaskInfo taskInfo) {
+    writeLock.lock();
+    try {
+      List<TaskInfo> tasksAtPriority = pendingTasks.get(taskInfo.priority);
+      if (tasksAtPriority == null) {
+        tasksAtPriority = new LinkedList<>();
+        pendingTasks.put(taskInfo.priority, tasksAtPriority);
+      }
+      tasksAtPriority.add(taskInfo);
+      knownTasks.putIfAbsent(taskInfo.task, taskInfo);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  /* Remove a task from the pending list */
+  private void removePendingTask(TaskInfo taskInfo) {
+    writeLock.lock();
+    try {
+      Priority priority = taskInfo.priority;
+      List<TaskInfo> taskInfoList = pendingTasks.get(priority);
+      if (taskInfoList == null || taskInfoList.isEmpty() || !taskInfoList.remove(taskInfo)) {
+        LOG.warn("Could not find task: " + taskInfo.task + " in pending list, at priority: "
+            + priority);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  /* Register a running task into the runningTasks structure */
+  private void registerRunningTask(TaskInfo taskInfo) {
+    writeLock.lock();
+    try {
+      int priority = taskInfo.priority.getPriority();
+      TreeSet<TaskInfo> tasksAtpriority = runningTasks.get(priority);
+      if (tasksAtpriority == null) {
+        tasksAtpriority = new TreeSet<>(TASK_INFO_COMPARATOR);
+        runningTasks.put(priority, tasksAtpriority);
+      }
+      tasksAtpriority.add(taskInfo);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  /* Unregister a task from the known and running structures */
+  private TaskInfo unregisterTask(Object task) {
+    writeLock.lock();
+    try {
+      TaskInfo taskInfo = knownTasks.remove(task);
+      if (taskInfo != null) {
+        if (taskInfo.assigned) {
+          // Remove from the running list.
+          int priority = taskInfo.priority.getPriority();
+          Set<TaskInfo> tasksAtPriority = runningTasks.get(priority);
+          Preconditions.checkState(tasksAtPriority != null,
+              "runningTasks should contain an entry if the task was in running state. Caused by task: {}", task);
+          tasksAtPriority.remove(taskInfo);
+          if (tasksAtPriority.isEmpty()) {
+            runningTasks.remove(priority);
+          }
+        }
+      } else {
+        LOG.warn("Could not find TaskInfo for task: {}. Not removing it from the running set", task);
+      }
+      return taskInfo;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private enum ScheduleResult {
+    // Successfully scheduled
+    SCHEDULED,
+
+    // Delayed to find a local match
+    DELAYED_LOCALITY,
+
+    // Delayed due to temporary resource availability
+    DELAYED_RESOURCES,
+
+    // Inadequate total resources - will never succeed / wait for new executors to become available
+    INADEQUATE_TOTAL_RESOURCES,
+  }
+
+  @VisibleForTesting
+  protected void schedulePendingTasks() {
+    writeLock.lock();
+    try {
+      Iterator<Entry<Priority, List<TaskInfo>>> pendingIterator =
+          pendingTasks.entrySet().iterator();
+      while (pendingIterator.hasNext()) {
+        Entry<Priority, List<TaskInfo>> entry = pendingIterator.next();
+        List<TaskInfo> taskListAtPriority = entry.getValue();
+        Iterator<TaskInfo> taskIter = taskListAtPriority.iterator();
+        boolean scheduledAllAtPriority = true;
+        while (taskIter.hasNext()) {
+
+          // TODO Optimization: Add a check to see if there's any capacity available. No point in
+          // walking through all active nodes, if they don't have potential capacity.
+
+          TaskInfo taskInfo = taskIter.next();
+          if (taskInfo.getNumPreviousAssignAttempts() == 1) {
+            dagStats.registerDelayedAllocation();
+          }
+          taskInfo.triedAssigningTask();
+          ScheduleResult scheduleResult = scheduleTask(taskInfo);
+          if (scheduleResult == ScheduleResult.SCHEDULED) {
+            taskIter.remove();
+          } else {
+            // TODO Handle INADEQUATE_TOTAL_RESOURCES eventually - either by throwin an error immediately,
+            // or waiting for some timeout for new executors and then throwing an error
+
+            // Try pre-empting a task so that a higher priority task can take it's place.
+            // Preempt only if there's no pending preemptions to avoid preempting twice for a task.
+            String[] potentialHosts;
+            if (scheduleResult == ScheduleResult.DELAYED_LOCALITY) {
+              // preempt only on specific hosts, if no preemptions already exist on those.
+              potentialHosts = taskInfo.requestedHosts;
+              //Protect against a bad location being requested.
+              if (potentialHosts == null || potentialHosts.length == 0) {
+                potentialHosts = null;
+              }
+            } else {
+              // preempt on any host.
+              potentialHosts = null;
+            }
+
+            if (potentialHosts != null) {
+              // Preempt on specific host
+              boolean shouldPreempt = true;
+              for (String host : potentialHosts) {
+                // Preempt only if there are not pending preemptions on the same host
+                // When the premption registers, the request at the highest priority will be given the slot,
+                // even if the initial request was for some other task.
+                // TODO Maybe register which task the preemption was for, to avoid a bad non-local allocation.
+                MutableInt pendingHostPreemptions = pendingPreemptionsPerHost.get(host);
+                if (pendingHostPreemptions != null && pendingHostPreemptions.intValue() > 0) {
+                  shouldPreempt = false;
+                  break;
+                }
+              }
+              if (shouldPreempt) {
+                LOG.info("Attempting to preempt for {}, pendingPreemptions={} on hosts={}",
+                    taskInfo.task, pendingPreemptions.get(), Arrays.toString(potentialHosts));
+                preemptTasks(entry.getKey().getPriority(), 1, potentialHosts);
+              }
+            } else {
+              // Request for a preemption if there's none pending. If a single preemption is pending,
+              // and this is the next task to be assigned, it will be assigned once that slot becomes available.
+              if (pendingPreemptions.get() == 0) {
+                LOG.info("Attempting to preempt for {}, pendingPreemptions={} on any host",
+                    taskInfo.task, pendingPreemptions.get());
+                preemptTasks(entry.getKey().getPriority(), 1, null);
+              }
+            }
+            // Since there was an allocation failure - don't try assigning tasks at the next priority.
+
+            scheduledAllAtPriority = false;
+            // Don't break if this allocation failure was a result of a LOCALITY_DELAY. Others could still be allocated.
+            if (scheduleResult != ScheduleResult.DELAYED_LOCALITY) {
+              break;
+            }
+          } // end of else - i.e. could not allocate
+        } // end of loop over pending tasks
+        if (taskListAtPriority.isEmpty()) {
+          // Remove the entry, if there's nothing left at the specific priority level
+          pendingIterator.remove();
+        }
+        if (!scheduledAllAtPriority) {
+          // Don't attempt scheduling for additional priorities
+          break;
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+
+  private ScheduleResult scheduleTask(TaskInfo taskInfo) {
+    SelectHostResult selectHostResult = selectHost(taskInfo);
+    if (selectHostResult.scheduleResult == ScheduleResult.SCHEDULED) {
+      NodeServiceInstancePair nsPair = selectHostResult.nodeServiceInstancePair;
+      Container container =
+          containerFactory.createContainer(resourcePerExecutor, taskInfo.priority,
+              nsPair.getServiceInstance().getHost(),
+              nsPair.getServiceInstance().getRpcPort());
+      writeLock.lock(); // While updating local structures
+      try {
+        LOG.info("Assigned task {} to container {}", taskInfo, container.getId());
+        dagStats.registerTaskAllocated(taskInfo.requestedHosts, taskInfo.requestedRacks,
+            nsPair.getServiceInstance().getHost());
+        taskInfo.setAssignmentInfo(nsPair.getServiceInstance(), container.getId(), clock.getTime());
+        registerRunningTask(taskInfo);
+        nsPair.getNodeInfo().registerTaskScheduled();
+      } finally {
+        writeLock.unlock();
+      }
+      getContext().taskAllocated(taskInfo.task, taskInfo.clientCookie, container);
+    }
+    return selectHostResult.scheduleResult;
+  }
+
+  // Removes tasks from the runningList and sends out a preempt request to the system.
+  // Subsequent tasks will be scheduled again once the de-allocate request for the preempted
+  // task is processed.
+  private void preemptTasks(int forPriority, int numTasksToPreempt, String []potentialHosts) {
+    Set<String> preemptHosts;
+    if (potentialHosts == null) {
+      preemptHosts = null;
+    } else {
+      preemptHosts = Sets.newHashSet(potentialHosts);
+    }
+    writeLock.lock();
+    List<TaskInfo> preemptedTaskList = null;
+    try {
+      NavigableMap<Integer, TreeSet<TaskInfo>> orderedMap = runningTasks.descendingMap();
+      Iterator<Entry<Integer, TreeSet<TaskInfo>>> iterator = orderedMap.entrySet().iterator();
+      int preemptedCount = 0;
+      while (iterator.hasNext() && preemptedCount < numTasksToPreempt) {
+        Entry<Integer, TreeSet<TaskInfo>> entryAtPriority = iterator.next();
+        if (entryAtPriority.getKey() > forPriority) {
+          Iterator<TaskInfo> taskInfoIterator = entryAtPriority.getValue().iterator();
+          while (taskInfoIterator.hasNext() && preemptedCount < numTasksToPreempt) {
+            TaskInfo taskInfo = taskInfoIterator.next();
+            if (preemptHosts == null || preemptHosts.contains(taskInfo.assignedInstance.getHost())) {
+              // Candidate for preemption.
+              preemptedCount++;
+              LOG.info("preempting {} for task at priority {} with potentialHosts={}", taskInfo,
+                  forPriority, potentialHosts == null ? "" : Arrays.toString(potentialHosts));
+              taskInfo.setPreemptedInfo(clock.getTime());
+              if (preemptedTaskList == null) {
+                preemptedTaskList = new LinkedList<>();
+              }
+              dagStats.registerTaskPreempted(taskInfo.assignedInstance.getHost());
+              preemptedTaskList.add(taskInfo);
+              registerPendingPreemption(taskInfo.assignedInstance.getHost());
+              // Remove from the runningTaskList
+              taskInfoIterator.remove();
+            }
+          }
+
+          // Remove entire priority level if it's been emptied.
+          if (entryAtPriority.getValue().isEmpty()) {
+            iterator.remove();
+          }
+        } else {
+          // No tasks qualify as preemptable
+          LOG.info("DBG: No tasks qualify as killable to schedule tasks at priority {}", forPriority);
+          break;
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+    // Send out the preempted request outside of the lock.
+    if (preemptedTaskList != null) {
+      for (TaskInfo taskInfo : preemptedTaskList) {
+        LOG.info("DBG: Preempting task {}", taskInfo);
+        getContext().preemptContainer(taskInfo.containerId);
+        // Preemption will finally be registered as a deallocateTask as a result of preemptContainer
+        // That resets preemption info and allows additional tasks to be pre-empted if required.
+      }
+    }
+    // The schedule loop will be triggered again when the deallocateTask request comes in for the
+    // preempted task.
+  }
+
+  private void registerPendingPreemption(String host) {
+    writeLock.lock();
+    try {
+      pendingPreemptions.incrementAndGet();
+      MutableInt val = pendingPreemptionsPerHost.get(host);
+      if (val == null) {
+        val = new MutableInt(1);
+        pendingPreemptionsPerHost.put(host, val);
+      }
+      val.increment();
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void unregisterPendingPreemption(String host) {
+    writeLock.lock();
+    try {
+      pendingPreemptions.decrementAndGet();
+      MutableInt val = pendingPreemptionsPerHost.get(host);
+      Preconditions.checkNotNull(val);
+      val.decrement();
+      // Not bothering with removing the entry. There's a limited number of hosts, and a good
+      // chance that the entry will make it back in when the AM is used for a long duration.
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private class NodeEnablerCallable implements Callable<Void> {
+
+    private AtomicBoolean isShutdown = new AtomicBoolean(false);
+    private static final long REFRESH_INTERVAL = 10000l;
+    long nextPollInterval = REFRESH_INTERVAL;
+    long lastRefreshTime;
+
+    @Override
+    public Void call() {
+
+      lastRefreshTime = System.currentTimeMillis();
+      while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
+        try {
+          while (true) {
+            NodeInfo nodeInfo = disabledNodesQueue.poll(nextPollInterval, TimeUnit.MILLISECONDS);
+            if (nodeInfo != null) {
+              long currentTime = System.currentTimeMillis();
+              // A node became available. Enable the node and try scheduling.
+              reenableDisabledNode(nodeInfo);
+              trySchedulingPendingTasks();
+
+              nextPollInterval -= (currentTime - lastRefreshTime);
+            }
+
+            if (nextPollInterval < 0 || nodeInfo == null) {
+              // timeout expired. Reset the poll interval and refresh nodes.
+              nextPollInterval = REFRESH_INTERVAL;
+              lastRefreshTime = System.currentTimeMillis();
+              // TODO Get rid of this polling once we have notificaitons from the registry sub-system
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("Refreshing instances based on poll interval");
+              }
+              refreshInstances();
+              scanForNodeChanges();
+            }
+          }
+        } catch (InterruptedException e) {
+          if (isShutdown.get()) {
+            LOG.info("NodeEnabler thread interrupted after shutdown");
+            break;
+          } else {
+            LOG.warn("NodeEnabler thread interrupted without being shutdown");
+            throw new RuntimeException("NodeEnabler thread interrupted without being shutdown", e);
+          }
+        }
+      }
+      return null;
+    }
+
+    // Call this first, then send in an interrupt to the thread.
+    public void shutdown() {
+      isShutdown.set(true);
+    }
+  }
+
+  private void trySchedulingPendingTasks() {
+    scheduleLock.lock();
+    try {
+      pendingScheduleInvodations.set(true);
+      scheduleCondition.signal();
+    } finally {
+      scheduleLock.unlock();
+    }
+  }
+
+  private class SchedulerCallable implements Callable<Void> {
+    private AtomicBoolean isShutdown = new AtomicBoolean(false);
+
+    @Override
+    public Void call() {
+      while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
+        scheduleLock.lock();
+        try {
+          while (!pendingScheduleInvodations.get()) {
+            scheduleCondition.await();
+          }
+        } catch (InterruptedException e) {
+          if (isShutdown.get()) {
+            LOG.info("Scheduler thread interrupted after shutdown");
+            break;
+          } else {
+            LOG.warn("Scheduler thread interrupted without being shutdown");
+            throw new RuntimeException("Scheduler thread interrupted without being shutdown", e);
+          }
+        } finally {
+          scheduleLock.unlock();
+        }
+
+        // Set pending to false since scheduling is about to run. Any triggers up to this point
+        // will be handled in the next run.
+        // A new request may come in right after this is set to false, but before the actual scheduling.
+        // This will be handled in this run, but will cause an immediate run after, which is harmless.
+        pendingScheduleInvodations.set(false);
+        // Schedule outside of the scheduleLock - which should only be used to wait on the condition.
+        schedulePendingTasks();
+      }
+      return null;
+    }
+
+    // Call this first, then send in an interrupt to the thread.
+    public void shutdown() {
+      isShutdown.set(true);
+    }
+  }
+
+  @VisibleForTesting
+  static class NodeInfo implements Delayed {
+    private final NodeBlacklistConf blacklistConf;
+    final ServiceInstance serviceInstance;
+    private final Clock clock;
+
+    long expireTimeMillis = -1;
+    private long numSuccessfulTasks = 0;
+    private long numSuccessfulTasksAtLastBlacklist = -1;
+    float cumulativeBackoffFactor = 1.0f;
+
+    // Indicates whether a node had a recent communication failure.
+    private boolean hadCommFailure = false;
+
+    // Indicates whether a node is disabled - for whatever reason - commFailure, busy, etc.
+    private boolean disabled = false;
+
+    private int numPreemptedTasks = 0;
+    private int numScheduledTasks = 0;
+    private final int numSchedulableTasks;
+
+
+    /**
+     * Create a NodeInfo bound to a service instance
+     *
+     * @param serviceInstance         the associated serviceInstance
+     * @param blacklistConf           blacklist configuration
+     * @param clock                   clock to use to obtain timing information
+     * @param numSchedulableTasksConf number of schedulable tasks on the node. 0 represents auto
+     *                                detect based on the serviceInstance, -1 indicates indicates
+     *                                unlimited capacity
+     */
+    NodeInfo(ServiceInstance serviceInstance, NodeBlacklistConf blacklistConf, Clock clock, int numSchedulableTasksConf) {
+      Preconditions.checkArgument(numSchedulableTasksConf >= -1, "NumSchedulableTasks must be >=-1");
+      this.serviceInstance = serviceInstance;
+      this.blacklistConf = blacklistConf;
+      this.clock = clock;
+
+      if (numSchedulableTasksConf == 0) {
+        int pendingQueueuCapacity = 0;
+        String pendingQueueCapacityString = serviceInstance.getProperties()
+            .get(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Setting up node: " + serviceInstance + ", with available capacity=" +
+              serviceInstance.getResource().getVirtualCores() + ", pendingQueueCapacity=" +
+              pendingQueueCapacityString);
+        }
+
+        if (pendingQueueCapacityString != null) {
+          pendingQueueuCapacity = Integer.parseInt(pendingQueueCapacityString);
+        }
+        this.numSchedulableTasks = serviceInstance.getResource().getVirtualCores() + pendingQueueuCapacity;
+      } else {
+        this.numSchedulableTasks = numSchedulableTasksConf;
+      }
+      LOG.info("Setting up node: " + serviceInstance + " with schedulableCapacity=" + this.numSchedulableTasks);
+    }
+
+    ServiceInstance getServiceInstance() {
+      return serviceInstance;
+    }
+
+    void enableNode() {
+      expireTimeMillis = -1;
+      disabled = false;
+      hadCommFailure = false;
+    }
+
+    void disableNode(boolean commFailure) {
+      long duration = blacklistConf.minDelay;
+      long currentTime = clock.getTime();
+      this.hadCommFailure = commFailure;
+      disabled = true;
+      if (numSuccessfulTasksAtLastBlacklist == numSuccessfulTasks) {
+        // Relying on a task succeeding to reset the exponent.
+        // There's no notifications on whether a task gets accepted or not. That would be ideal to
+        // reset this.
+        cumulativeBackoffFactor = cumulativeBackoffFactor * blacklistConf.backoffFactor;
+      } else {
+        // Was able to execute something before the last blacklist. Reset the exponent.
+        cumulativeBackoffFactor = 1.0f;
+      }
+
+      long delayTime = (long) (duration * cumulativeBackoffFactor);
+      if (delayTime > blacklistConf.maxDelay) {
+        delayTime = blacklistConf.maxDelay;
+      }
+      if (LOG.isInfoEnabled()) {
+        LOG.info("Disabling instance " + serviceInstance + " for " + delayTime + " milli-seconds");
+      }
+      expireTimeMillis = currentTime + delayTime;
+      numSuccessfulTasksAtLastBlacklist = numSuccessfulTasks;
+    }
+
+    void registerTaskScheduled() {
+      numScheduledTasks++;
+    }
+
+    void registerTaskSuccess() {
+      numSuccessfulTasks++;
+      numScheduledTasks--;
+    }
+
+    void registerUnsuccessfulTaskEnd(boolean wasPreempted) {
+      numScheduledTasks--;
+      if (wasPreempted) {
+        numPreemptedTasks++;
+      }
+    }
+
+    public boolean isDisabled() {
+      return disabled;
+    }
+
+    public boolean hadCommFailure() {
+      return hadCommFailure;
+    }
+    /* Returning true does not guarantee that the task will run, considering other queries
+    may be running in the system. Also depends upon the capacity usage configuration
+     */
+    public boolean canAcceptTask() {
+      boolean result = !hadCommFailure && !disabled && serviceInstance.isAlive()
+          &&(numSchedulableTasks == -1 || ((numSchedulableTasks - numScheduledTasks) > 0));
+      LOG.info("canAcceptTask={}, numScheduledTasks={}, numSchedulableTasks={}, hadCommFailure={}, disabled={}, serviceInstance.isAlive={}", result, numScheduledTasks, numSchedulableTasks, hadCommFailure, disabled, serviceInstance.isAlive());
+      return result;
+    }
+
+    @Override
+    public long getDelay(TimeUnit unit) {
+      return unit.convert(expireTimeMillis - clock.getTime(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public int compareTo(Delayed o) {
+      NodeInfo other = (NodeInfo) o;
+      if (other.expireTimeMillis > this.expireTimeMillis) {
+        return -1;
+      } else if (other.expireTimeMillis < this.expireTimeMillis) {
+        return 1;
+      } else {
+        return 0;
+      }
+    }
+
+    @Override
+    public String toString() {
+      return "NodeInfo{" + "instance=" + serviceInstance
+          + ", expireTimeMillis=" + expireTimeMillis + ", numSuccessfulTasks=" + numSuccessfulTasks
+          + ", numSuccessfulTasksAtLastBlacklist=" + numSuccessfulTasksAtLastBlacklist
+          + ", cumulativeBackoffFactor=" + cumulativeBackoffFactor
+          + ", numSchedulableTasks=" + numSchedulableTasks
+          + ", numScheduledTasks=" + numScheduledTasks
+          + ", disabled=" + disabled
+          + ", commFailures=" + hadCommFailure
+          +'}';
+    }
+  }
+
+  @VisibleForTesting
+  static class StatsPerDag {
+    int numRequestedAllocations = 0;
+    int numRequestsWithLocation = 0;
+    int numRequestsWithoutLocation = 0;
+    int numTotalAllocations = 0;
+    int numLocalAllocations = 0;
+    int numNonLocalAllocations = 0;
+    int numAllocationsNoLocalityRequest = 0;
+    int numRejectedTasks = 0;
+    int numCommFailures = 0;
+    int numDelayedAllocations = 0;
+    int numPreemptedTasks = 0;
+    Map<String, AtomicInteger> localityBasedNumAllocationsPerHost = new HashMap<>();
+    Map<String, AtomicInteger> numAllocationsPerHost = new HashMap<>();
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder();
+      sb.append("NumPreemptedTasks=").append(numPreemptedTasks).append(", ");
+      sb.append("NumRequestedAllocations=").append(numRequestedAllocations).append(", ");
+      sb.append("NumRequestsWithlocation=").append(numRequestsWithLocation).append(", ");
+      sb.append("NumLocalAllocations=").append(numLocalAllocations).append(",");
+      sb.append("NumNonLocalAllocations=").append(numNonLocalAllocations).append(",");
+      sb.append("NumTotalAllocations=").append(numTotalAllocations).append(",");
+      sb.append("NumRequestsWithoutLocation=").append(numRequestsWithoutLocation).append(", ");
+      sb.append("NumRejectedTasks=").append(numRejectedTasks).append(", ");
+      sb.append("NumCommFailures=").append(numCommFailures).append(", ");
+      sb.append("NumDelayedAllocations=").append(numDelayedAllocations).append(", ");
+      sb.append("LocalityBasedAllocationsPerHost=").append(localityBasedNumAllocationsPerHost)
+          .append(", ");
+      sb.append("NumAllocationsPerHost=").append(numAllocationsPerHost);
+      return sb.toString();
+    }
+
+    void registerTaskRequest(String[] requestedHosts, String[] requestedRacks) {
+      numRequestedAllocations++;
+      // TODO Change after HIVE-9987. For now, there's no rack matching.
+      if (requestedHosts != null && requestedHosts.length != 0) {
+        numRequestsWithLocation++;
+      } else {
+        numRequestsWithoutLocation++;
+      }
+    }
+
+    void registerTaskAllocated(String[] requestedHosts, String[] requestedRacks,
+        String allocatedHost) {
+      // TODO Change after HIVE-9987. For now, there's no rack matching.
+      if (requestedHosts != null && requestedHosts.length != 0) {
+        Set<String> requestedHostSet = new HashSet<>(Arrays.asList(requestedHosts));
+        if (requestedHostSet.contains(allocatedHost)) {
+          numLocalAllocations++;
+          _registerAllocationInHostMap(allocatedHost, localityBasedNumAllocationsPerHost);
+        } else {
+          numNonLocalAllocations++;
+        }
+      } else {
+        numAllocationsNoLocalityRequest++;
+      }
+      numTotalAllocations++;
+      _registerAllocationInHostMap(allocatedHost, numAllocationsPerHost);
+    }
+
+    void registerTaskPreempted(String host) {
+      numPreemptedTasks++;
+    }
+
+    void registerCommFailure(String host) {
+      numCommFailures++;
+    }
+
+    void registerTaskRejected(String host) {
+      numRejectedTasks++;
+    }
+
+    void registerDelayedAllocation() {
+      numDelayedAllocations++;
+    }
+
+    private void _registerAllocationInHostMap(String host, Map<String, AtomicInteger> hostMap) {
+      AtomicInteger val = hostMap.get(host);
+      if (val == null) {
+        val = new AtomicInteger(0);
+        hostMap.put(host, val);
+      }
+      val.incrementAndGet();
+    }
+  }
+
+  private static class TaskInfo {
+    // IDs used to ensure two TaskInfos are different without using the underlying task instance.
+    // Required for insertion into a TreeMap
+    static final AtomicLong ID_GEN = new AtomicLong(0);
+    final long uniqueId;
+    final Object task;
+    final Object clientCookie;
+    final Priority priority;
+    final Resource capability;
+    final String[] requestedHosts;
+    final String[] requestedRacks;
+    final long requestTime;
+    long startTime;
+    long preemptTime;
+    ContainerId containerId;
+    ServiceInstance assignedInstance;
+    private boolean assigned = false;
+    private boolean preempted = false;
+
+    private int numAssignAttempts = 0;
+
+    // TaskInfo instances for two different tasks will not be the same. Only a single instance should
+    // ever be created for a taskAttempt
+    public TaskInfo(Object task, Object clientCookie, Priority priority, Resource capability,
+        String[] hosts, String[] racks, long requestTime) {
+      this.task = task;
+      this.clientCookie = clientCookie;
+      this.priority = priority;
+      this.capability = capability;
+      this.requestedHosts = hosts;
+      this.requestedRacks = racks;
+      this.requestTime = requestTime;
+      this.uniqueId = ID_GEN.getAndIncrement();
+    }
+
+    void setAssignmentInfo(ServiceInstance instance, ContainerId containerId, long startTime) {
+      this.assignedInstance = instance;
+        this.containerId = containerId;
+      this.startTime = startTime;
+      assigned = true;
+    }
+
+    void setPreemptedInfo(long preemptTime) {
+      this.preempted = true;
+      this.assigned = false;
+      this.preemptTime = preemptTime;
+    }
+
+    void triedAssigningTask() {
+      numAssignAttempts++;
+    }
+
+    int getNumPreviousAssignAttempts() {
+      return numAssignAttempts;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+
+      TaskInfo taskInfo = (TaskInfo) o;
+
+      if (uniqueId != taskInfo.uniqueId) {
+        return false;
+      }
+      return task.equals(taskInfo.task);
+
+    }
+
+    @Override
+    public int hashCode() {
+      int result = (int) (uniqueId ^ (uniqueId >>> 32));
+      result = 31 * result + task.hashCode();
+      return result;
+    }
+
+    @Override
+    public String toString() {
+      return "TaskInfo{" +
+          "task=" + task +
+          ", priority=" + priority +
+          ", startTime=" + startTime +
+          ", containerId=" + containerId +
+          ", assignedInstance=" + assignedInstance +
+          ", uniqueId=" + uniqueId +
+          '}';
+    }
+  }
+
+  // Newer tasks first.
+  private static class TaskStartComparator implements Comparator<TaskInfo> {
+
+    @Override
+    public int compare(TaskInfo o1, TaskInfo o2) {
+      if (o1.startTime > o2.startTime) {
+        return -1;
+      } else if (o1.startTime < o2.startTime) {
+        return 1;
+      } else {
+        // Comparing on time is not sufficient since two may be created at the same time,
+        // in which case inserting into a TreeSet/Map would break
+        if (o1.uniqueId > o2.uniqueId) {
+          return -1;
+        } else if (o1.uniqueId < o2.uniqueId) {
+          return 1;
+        } else {
+          return 0;
+        }
+      }
+    }
+  }
+
+  private static class SelectHostResult {
+    final NodeServiceInstancePair nodeServiceInstancePair;
+    final ScheduleResult scheduleResult;
+
+    SelectHostResult(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
+      this.nodeServiceInstancePair = new NodeServiceInstancePair(serviceInstance, nodeInfo);
+      this.scheduleResult = ScheduleResult.SCHEDULED;
+    }
+
+    SelectHostResult(ScheduleResult scheduleResult) {
+      this.nodeServiceInstancePair = null;
+      this.scheduleResult = scheduleResult;
+    }
+  }
+
+  private static final SelectHostResult SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY =
+      new SelectHostResult(ScheduleResult.INADEQUATE_TOTAL_RESOURCES);
+  private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_LOCALITY =
+      new SelectHostResult(ScheduleResult.DELAYED_LOCALITY);
+  private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_RESOURCES =
+      new SelectHostResult(ScheduleResult.DELAYED_RESOURCES);
+
+  private static class NodeServiceInstancePair {
+    final NodeInfo nodeInfo;
+    final ServiceInstance serviceInstance;
+
+    private NodeServiceInstancePair(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
+      this.serviceInstance = serviceInstance;
+      this.nodeInfo = nodeInfo;
+    }
+
+    public ServiceInstance getServiceInstance() {
+      return serviceInstance;
+    }
+
+    public NodeInfo getNodeInfo() {
+      return nodeInfo;
+    }
+  }
+
+  private static final class NodeBlacklistConf {
+    private final long minDelay;
+    private final long maxDelay;
+    private final float backoffFactor;
+
+    public NodeBlacklistConf(long minDelay, long maxDelay, float backoffFactor) {
+      this.minDelay = minDelay;
+      this.maxDelay = maxDelay;
+      this.backoffFactor = backoffFactor;
+    }
+
+    @Override
+    public String toString() {
+      return "NodeBlacklistConf{" +
+          "minDelay=" + minDelay +
+          ", maxDelay=" + maxDelay +
+          ", backoffFactor=" + backoffFactor +
+          '}';
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java
new file mode 100644
index 0000000..2c3e53c
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.mapreduce.input.MultiMRInput;
+
+@InterfaceAudience.Private
+public class LlapTezUtils {
+  public static boolean isSourceOfInterest(String inputClassName) {
+    // MRInput is not of interest since it'll always be ready.
+    return !(inputClassName.equals(MRInputLegacy.class.getName()) || inputClassName.equals(
+        MultiMRInput.class.getName()) || inputClassName.equals(MRInput.class.getName()));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
new file mode 100644
index 0000000..4102f5b
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
+import org.apache.hadoop.security.authorize.Service;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
+
+public class LlapUmbilicalPolicyProvider extends TezAMPolicyProvider {
+
+  private static Service[] services;
+  private static final Object servicesLock = new Object();
+
+  @Override
+  public Service[] getServices() {
+    if (services != null) return services;
+    synchronized (servicesLock) {
+      if (services != null) return services;
+      Service[] parentSvc = super.getServices();
+      Service[] result = Arrays.copyOf(parentSvc, parentSvc.length + 1);
+      result[parentSvc.length] =  new Service(
+          TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL,
+          LlapTaskUmbilicalProtocol.class);
+      return (services = result);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
new file mode 100644
index 0000000..d8f7574
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins.helpers;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
+import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.tez.Converters;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.runtime.api.impl.InputSpec;
+
+public class SourceStateTracker {
+
+  private static final Logger LOG = LoggerFactory.getLogger(SourceStateTracker.class);
+
+  private final TaskCommunicatorContext taskCommunicatorContext;
+  private final LlapTaskCommunicator taskCommunicator;
+
+  private final QueryIdentifierProto BASE_QUERY_IDENTIFIER;
+
+  // Tracks vertices for which notifications have been registered
+  private final Set<String> notificationRegisteredVertices = new HashSet<>();
+
+  private final Map<String, SourceInfo> sourceInfoMap = new HashMap<>();
+  private final Map<LlapNodeId, NodeInfo> nodeInfoMap = new HashMap<>();
+
+  private volatile QueryIdentifierProto currentQueryIdentifier;
+
+  public SourceStateTracker(TaskCommunicatorContext taskCommunicatorContext,
+                            LlapTaskCommunicator taskCommunicator) {
+    this.taskCommunicatorContext = taskCommunicatorContext;
+    this.taskCommunicator = taskCommunicator;
+    BASE_QUERY_IDENTIFIER = QueryIdentifierProto.newBuilder()
+        .setAppIdentifier(taskCommunicatorContext.getCurrentAppIdentifier()).build();
+  }
+
+  /**
+   * To be invoked after each DAG completes.
+   */
+  public synchronized void resetState(int newDagId) {
+    sourceInfoMap.clear();
+    nodeInfoMap.clear();
+    notificationRegisteredVertices.clear();
+    this.currentQueryIdentifier =
+        QueryIdentifierProto.newBuilder(BASE_QUERY_IDENTIFIER).setDagIdentifier(newDagId).build();
+  }
+
+  /**
+   * Used to register a task for state updates. Effectively registers for state updates to go to the specific node.
+   * @param host
+   * @param port
+   * @param inputSpecList
+   */
+  public synchronized void registerTaskForStateUpdates(String host, int port,
+                                                       List<InputSpec> inputSpecList) {
+
+    // Add tracking information. Check if source state already known and send out an update if it is.
+
+    List<String> sourcesOfInterest = getSourceInterestList(inputSpecList);
+    if (sourcesOfInterest != null && !sourcesOfInterest.isEmpty()) {
+      LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+      NodeInfo nodeInfo = getNodeInfo(nodeId);
+
+      // Set up the data structures, before any notifications come in.
+      for (String src : sourcesOfInterest) {
+        VertexState oldStateForNode = nodeInfo.getLastKnownStateForSource(src);
+        if (oldStateForNode == null) {
+          // Not registered for this node.
+          // Register and send state if it is successful.
+          SourceInfo srcInfo = getSourceInfo(src);
+          srcInfo.addNode(nodeId);
+
+          nodeInfo.addSource(src, srcInfo.lastKnownState);
+          if (srcInfo.lastKnownState == VertexState.SUCCEEDED) {
+            sendStateUpdateToNode(nodeId, src, srcInfo.lastKnownState);
+          }
+
+        } else {
+          // Already registered to send updates to this node for the specific source.
+          // Nothing to do for now, unless tracking tasks at a later point.
+        }
+
+        // Setup for actual notifications, if not already done for a previous task.
+        maybeRegisterForVertexUpdates(src);
+      }
+    } else {
+      // Don't need to track anything for this task. No new notifications, etc.
+    }
+  }
+
+  /**
+   * Handled notifications on state updates for sources
+   * @param sourceName
+   * @param sourceState
+   */
+  public synchronized void sourceStateUpdated(String sourceName, VertexState sourceState) {
+    SourceInfo sourceInfo = getSourceInfo(sourceName);
+    // Update source info if the state is SUCCEEDED
+    if (sourceState == VertexState.SUCCEEDED) {
+      sourceInfo.numCompletedTasks = taskCommunicatorContext.getVertexCompletedTaskCount(sourceName);
+      sourceInfo.numTasks = taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
+    }
+    sourceInfo.lastKnownState = sourceState;
+    // Checking state per node for future failure handling scenarios, where an update
+    // to a single node may fail.
+    for (LlapNodeId nodeId : sourceInfo.getInterestedNodes()) {
+      NodeInfo nodeInfo = nodeInfoMap.get(nodeId);
+      VertexState lastStateForNode = nodeInfo.getLastKnownStateForSource(sourceName);
+      // Send only if the state has changed.
+      if (lastStateForNode != sourceState) {
+        nodeInfo.setLastKnownStateForSource(sourceName, sourceState);
+        sendStateUpdateToNode(nodeId, sourceName, sourceState);
+      }
+    }
+  }
+
+
+  // Assumes serialized DAGs within an AM, and a reset of structures after each DAG completes.
+  /**
+   * Constructs FragmentRuntimeInfo for scheduling within LLAP daemons.
+   * Also caches state based on state updates.
+   * @param vertexName
+   * @param fragmentNumber
+   * @param priority
+   * @return
+   */
+  public synchronized FragmentRuntimeInfo getFragmentRuntimeInfo(String vertexName, int fragmentNumber,
+                                                                 int priority) {
+    FragmentRuntimeInfo.Builder builder = FragmentRuntimeInfo.newBuilder();
+    maybeRegisterForVertexUpdates(vertexName);
+
+    MutableInt totalTaskCount = new MutableInt(0);
+    MutableInt completedTaskCount = new MutableInt(0);
+    computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, vertexName);
+
+    builder.setNumSelfAndUpstreamCompletedTasks(completedTaskCount.intValue());
+    builder.setNumSelfAndUpstreamTasks(totalTaskCount.intValue());
+    builder.setDagStartTime(taskCommunicatorContext.getDagStartTime());
+    builder.setWithinDagPriority(priority);
+    builder.setFirstAttemptStartTime(taskCommunicatorContext.getFirstAttemptStartTime(vertexName, fragmentNumber));
+    builder.setCurrentAttemptStartTime(System.currentTimeMillis());
+    return builder.build();
+  }
+
+  private void computeUpstreamTaskCounts(MutableInt completedTaskCount, MutableInt totalTaskCount, String sourceName) {
+    SourceInfo sourceInfo = getSourceInfo(sourceName);
+    if (sourceInfo.lastKnownState == VertexState.SUCCEEDED) {
+      // Some of the information in the source is complete. Don't need to fetch it from the context.
+      completedTaskCount.add(sourceInfo.numCompletedTasks);
+      totalTaskCount.add(sourceInfo.numTasks);
+    } else {
+      completedTaskCount.add(taskCommunicatorContext.getVertexCompletedTaskCount(sourceName));
+      int totalCount =taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
+      // Uninitialized vertices will report count as 0.
+      totalCount = totalCount == -1 ? 0 : totalCount;
+      totalTaskCount.add(totalCount);
+    }
+
+    // Walk through all the source vertices
+    for (String up : taskCommunicatorContext.getInputVertexNames(sourceName)) {
+      computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, up);
+    }
+  }
+
+  private static class SourceInfo {
+
+    // Always start in the running state. Requests for state updates will be sent out after registration.
+    private VertexState lastKnownState = VertexState.RUNNING;
+
+    // Used for sending notifications about a vertex completed. For canFinish
+    // Can be converted to a Tez event, if this is sufficient to decide on pre-emption
+    private final List<LlapNodeId> interestedNodes = new LinkedList<>();
+
+    // Used for sending information for scheduling priority.
+    private int numTasks;
+    private int numCompletedTasks;
+
+    void addNode(LlapNodeId nodeId) {
+      interestedNodes.add(nodeId);
+    }
+
+    List<LlapNodeId> getInterestedNodes() {
+      return this.interestedNodes;
+    }
+
+
+
+  }
+
+  private synchronized SourceInfo getSourceInfo(String srcName) {
+    SourceInfo sourceInfo = sourceInfoMap.get(srcName);
+    if (sourceInfo == null) {
+      sourceInfo = new SourceInfo();
+      sourceInfoMap.put(srcName, sourceInfo);
+    }
+    return sourceInfo;
+  }
+
+
+  private static class NodeInfo {
+    private final Map<String, VertexState> sourcesOfInterest = new HashMap<>();
+
+    void addSource(String srcName, VertexState sourceState) {
+      sourcesOfInterest.put(srcName, sourceState);
+    }
+
+    VertexState getLastKnownStateForSource(String src) {
+      return sourcesOfInterest.get(src);
+    }
+
+    void setLastKnownStateForSource(String src, VertexState state) {
+      sourcesOfInterest.put(src, state);
+    }
+  }
+
+  private synchronized NodeInfo getNodeInfo(LlapNodeId llapNodeId) {
+    NodeInfo nodeInfo = nodeInfoMap.get(llapNodeId);
+    if (nodeInfo == null) {
+      nodeInfo = new NodeInfo();
+      nodeInfoMap.put(llapNodeId, nodeInfo);
+    }
+    return nodeInfo;
+  }
+
+
+  private List<String> getSourceInterestList(List<InputSpec> inputSpecList) {
+    List<String> sourcesOfInterest = Collections.emptyList();
+    if (inputSpecList != null) {
+      boolean alreadyFound = false;
+      for (InputSpec inputSpec : inputSpecList) {
+        if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) {
+          if (!alreadyFound) {
+            alreadyFound = true;
+            sourcesOfInterest = new LinkedList<>();
+          }
+          sourcesOfInterest.add(inputSpec.getSourceVertexName());
+        }
+      }
+    }
+    return sourcesOfInterest;
+  }
+
+
+  private void maybeRegisterForVertexUpdates(String sourceName) {
+    if (!notificationRegisteredVertices.contains(sourceName)) {
+      notificationRegisteredVertices.add(sourceName);
+      taskCommunicatorContext.registerForVertexStateUpdates(sourceName, EnumSet.of(
+          VertexState.RUNNING, VertexState.SUCCEEDED));
+    }
+  }
+
+
+
+  void sendStateUpdateToNode(LlapNodeId nodeId, String sourceName, VertexState state) {
+    taskCommunicator.sendStateUpdate(nodeId.getHostname(), nodeId.getPort(),
+        SourceStateUpdatedRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifier)
+            .setSrcName(sourceName).setState(Converters.fromVertexState(state)).build());
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
new file mode 100644
index 0000000..8f3d104
--- /dev/null
+++ b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.junit.Test;
+
+public class TestLlapTaskCommunicator {
+
+  @Test (timeout = 5000)
+  public void testEntityTracker1() {
+    LlapTaskCommunicator.EntityTracker entityTracker = new LlapTaskCommunicator.EntityTracker();
+
+    String host1 = "host1";
+    String host2 = "host2";
+    String host3 = "host3";
+    int port = 1451;
+
+
+    // Simple container registration and un-registration without any task attempt being involved.
+    ContainerId containerId101 = constructContainerId(101);
+    entityTracker.registerContainer(containerId101, host1, port);
+    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId101));
+
+    entityTracker.unregisterContainer(containerId101);
+    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
+    assertNull(entityTracker.getNodeIdForContainer(containerId101));
+    assertEquals(0, entityTracker.nodeMap.size());
+    assertEquals(0, entityTracker.attemptToNodeMap.size());
+    assertEquals(0, entityTracker.containerToNodeMap.size());
+
+
+    // Simple task registration and un-registration.
+    ContainerId containerId1 = constructContainerId(1);
+    TezTaskAttemptID taskAttemptId1 = constructTaskAttemptId(1);
+    entityTracker.registerTaskAttempt(containerId1, taskAttemptId1, host1, port);
+    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId1));
+    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
+
+    entityTracker.unregisterTaskAttempt(taskAttemptId1);
+    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
+    assertNull(entityTracker.getNodeIdForContainer(containerId1));
+    assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
+    assertEquals(0, entityTracker.nodeMap.size());
+    assertEquals(0, entityTracker.attemptToNodeMap.size());
+    assertEquals(0, entityTracker.containerToNodeMap.size());
+
+    // Register taskAttempt, unregister container. TaskAttempt should also be unregistered
+    ContainerId containerId201 = constructContainerId(201);
+    TezTaskAttemptID taskAttemptId201 = constructTaskAttemptId(201);
+    entityTracker.registerTaskAttempt(containerId201, taskAttemptId201, host1, port);
+    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId201));
+    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
+
+    entityTracker.unregisterContainer(containerId201);
+    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
+    assertNull(entityTracker.getNodeIdForContainer(containerId201));
+    assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
+    assertEquals(0, entityTracker.nodeMap.size());
+    assertEquals(0, entityTracker.attemptToNodeMap.size());
+    assertEquals(0, entityTracker.containerToNodeMap.size());
+
+    entityTracker.unregisterTaskAttempt(taskAttemptId201); // No errors
+  }
+
+
+  private ContainerId constructContainerId(int id) {
+    ContainerId containerId = mock(ContainerId.class);
+    doReturn(id).when(containerId).getId();
+    doReturn((long)id).when(containerId).getContainerId();
+    return containerId;
+  }
+
+  private TezTaskAttemptID constructTaskAttemptId(int id) {
+    TezTaskAttemptID taskAttemptId = mock(TezTaskAttemptID.class);
+    doReturn(id).when(taskAttemptId).getId();
+    return taskAttemptId;
+  }
+
+}


[09/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
----------------------------------------------------------------------
diff --git a/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java b/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
new file mode 100644
index 0000000..4ab7b32
--- /dev/null
+++ b/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
@@ -0,0 +1,14553 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: LlapDaemonProtocol.proto
+
+package org.apache.hadoop.hive.llap.daemon.rpc;
+
+public final class LlapDaemonProtocolProtos {
+  private LlapDaemonProtocolProtos() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  /**
+   * Protobuf enum {@code SourceStateProto}
+   */
+  public enum SourceStateProto
+      implements com.google.protobuf.ProtocolMessageEnum {
+    /**
+     * <code>S_SUCCEEDED = 1;</code>
+     */
+    S_SUCCEEDED(0, 1),
+    /**
+     * <code>S_RUNNING = 2;</code>
+     */
+    S_RUNNING(1, 2),
+    ;
+
+    /**
+     * <code>S_SUCCEEDED = 1;</code>
+     */
+    public static final int S_SUCCEEDED_VALUE = 1;
+    /**
+     * <code>S_RUNNING = 2;</code>
+     */
+    public static final int S_RUNNING_VALUE = 2;
+
+
+    public final int getNumber() { return value; }
+
+    public static SourceStateProto valueOf(int value) {
+      switch (value) {
+        case 1: return S_SUCCEEDED;
+        case 2: return S_RUNNING;
+        default: return null;
+      }
+    }
+
+    public static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>() {
+            public SourceStateProto findValueByNumber(int number) {
+              return SourceStateProto.valueOf(number);
+            }
+          };
+
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(0);
+    }
+
+    private static final SourceStateProto[] VALUES = values();
+
+    public static SourceStateProto valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+
+    private final int index;
+    private final int value;
+
+    private SourceStateProto(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+
+    // @@protoc_insertion_point(enum_scope:SourceStateProto)
+  }
+
+  /**
+   * Protobuf enum {@code SubmissionStateProto}
+   */
+  public enum SubmissionStateProto
+      implements com.google.protobuf.ProtocolMessageEnum {
+    /**
+     * <code>ACCEPTED = 1;</code>
+     */
+    ACCEPTED(0, 1),
+    /**
+     * <code>REJECTED = 2;</code>
+     */
+    REJECTED(1, 2),
+    /**
+     * <code>EVICTED_OTHER = 3;</code>
+     */
+    EVICTED_OTHER(2, 3),
+    ;
+
+    /**
+     * <code>ACCEPTED = 1;</code>
+     */
+    public static final int ACCEPTED_VALUE = 1;
+    /**
+     * <code>REJECTED = 2;</code>
+     */
+    public static final int REJECTED_VALUE = 2;
+    /**
+     * <code>EVICTED_OTHER = 3;</code>
+     */
+    public static final int EVICTED_OTHER_VALUE = 3;
+
+
+    public final int getNumber() { return value; }
+
+    public static SubmissionStateProto valueOf(int value) {
+      switch (value) {
+        case 1: return ACCEPTED;
+        case 2: return REJECTED;
+        case 3: return EVICTED_OTHER;
+        default: return null;
+      }
+    }
+
+    public static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>() {
+            public SubmissionStateProto findValueByNumber(int number) {
+              return SubmissionStateProto.valueOf(number);
+            }
+          };
+
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(1);
+    }
+
+    private static final SubmissionStateProto[] VALUES = values();
+
+    public static SubmissionStateProto valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+
+    private final int index;
+    private final int value;
+
+    private SubmissionStateProto(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+
+    // @@protoc_insertion_point(enum_scope:SubmissionStateProto)
+  }
+
+  public interface UserPayloadProtoOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+
+    // optional bytes user_payload = 1;
+    /**
+     * <code>optional bytes user_payload = 1;</code>
+     */
+    boolean hasUserPayload();
+    /**
+     * <code>optional bytes user_payload = 1;</code>
+     */
+    com.google.protobuf.ByteString getUserPayload();
+
+    // optional int32 version = 2;
+    /**
+     * <code>optional int32 version = 2;</code>
+     */
+    boolean hasVersion();
+    /**
+     * <code>optional int32 version = 2;</code>
+     */
+    int getVersion();
+  }
+  /**
+   * Protobuf type {@code UserPayloadProto}
+   */
+  public static final class UserPayloadProto extends
+      com.google.protobuf.GeneratedMessage
+      implements UserPayloadProtoOrBuilder {
+    // Use UserPayloadProto.newBuilder() to construct.
+    private UserPayloadProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+      super(builder);
+      this.unknownFields = builder.getUnknownFields();
+    }
+    private UserPayloadProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final UserPayloadProto defaultInstance;
+    public static UserPayloadProto getDefaultInstance() {
+      return defaultInstance;
+    }
+
+    public UserPayloadProto getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @java.lang.Override
+    public final com.google.protobuf.UnknownFieldSet
+        getUnknownFields() {
+      return this.unknownFields;
+    }
+    private UserPayloadProto(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      initFields();
+      int mutable_bitField0_ = 0;
+      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder();
+      try {
+        boolean done = false;
+        while (!done) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              done = true;
+              break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              userPayload_ = input.readBytes();
+              break;
+            }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              version_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+        throw e.setUnfinishedMessage(this);
+      } catch (java.io.IOException e) {
+        throw new com.google.protobuf.InvalidProtocolBufferException(
+            e.getMessage()).setUnfinishedMessage(this);
+      } finally {
+        this.unknownFields = unknownFields.build();
+        makeExtensionsImmutable();
+      }
+    }
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
+    }
+
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
+    }
+
+    public static com.google.protobuf.Parser<UserPayloadProto> PARSER =
+        new com.google.protobuf.AbstractParser<UserPayloadProto>() {
+      public UserPayloadProto parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new UserPayloadProto(input, extensionRegistry);
+      }
+    };
+
+    @java.lang.Override
+    public com.google.protobuf.Parser<UserPayloadProto> getParserForType() {
+      return PARSER;
+    }
+
+    private int bitField0_;
+    // optional bytes user_payload = 1;
+    public static final int USER_PAYLOAD_FIELD_NUMBER = 1;
+    private com.google.protobuf.ByteString userPayload_;
+    /**
+     * <code>optional bytes user_payload = 1;</code>
+     */
+    public boolean hasUserPayload() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * <code>optional bytes user_payload = 1;</code>
+     */
+    public com.google.protobuf.ByteString getUserPayload() {
+      return userPayload_;
+    }
+
+    // optional int32 version = 2;
+    public static final int VERSION_FIELD_NUMBER = 2;
+    private int version_;
+    /**
+     * <code>optional int32 version = 2;</code>
+     */
+    public boolean hasVersion() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * <code>optional int32 version = 2;</code>
+     */
+    public int getVersion() {
+      return version_;
+    }
+
+    private void initFields() {
+      userPayload_ = com.google.protobuf.ByteString.EMPTY;
+      version_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, userPayload_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeInt32(2, version_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, userPayload_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(2, version_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) obj;
+
+      boolean result = true;
+      result = result && (hasUserPayload() == other.hasUserPayload());
+      if (hasUserPayload()) {
+        result = result && getUserPayload()
+            .equals(other.getUserPayload());
+      }
+      result = result && (hasVersion() == other.hasVersion());
+      if (hasVersion()) {
+        result = result && (getVersion()
+            == other.getVersion());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+
+    private int memoizedHashCode = 0;
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasUserPayload()) {
+        hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
+        hash = (53 * hash) + getUserPayload().hashCode();
+      }
+      if (hasVersion()) {
+        hash = (37 * hash) + VERSION_FIELD_NUMBER;
+        hash = (53 * hash) + getVersion();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return PARSER.parseDelimitedFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input, extensionRegistry);
+    }
+
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /**
+     * Protobuf type {@code UserPayloadProto}
+     */
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
+      }
+
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
+      }
+
+      // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+
+      private Builder(
+          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+
+      public Builder clear() {
+        super.clear();
+        userPayload_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        version_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getDefaultInstanceForType() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto build() {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto buildPartial() {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.userPayload_ = userPayload_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.version_ = version_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) {
+          return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other) {
+        if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) return this;
+        if (other.hasUserPayload()) {
+          setUserPayload(other.getUserPayload());
+        }
+        if (other.hasVersion()) {
+          setVersion(other.getVersion());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parsedMessage = null;
+        try {
+          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) e.getUnfinishedMessage();
+          throw e;
+        } finally {
+          if (parsedMessage != null) {
+            mergeFrom(parsedMessage);
+          }
+        }
+        return this;
+      }
+      private int bitField0_;
+
+      // optional bytes user_payload = 1;
+      private com.google.protobuf.ByteString userPayload_ = com.google.protobuf.ByteString.EMPTY;
+      /**
+       * <code>optional bytes user_payload = 1;</code>
+       */
+      public boolean hasUserPayload() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * <code>optional bytes user_payload = 1;</code>
+       */
+      public com.google.protobuf.ByteString getUserPayload() {
+        return userPayload_;
+      }
+      /**
+       * <code>optional bytes user_payload = 1;</code>
+       */
+      public Builder setUserPayload(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        userPayload_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional bytes user_payload = 1;</code>
+       */
+      public Builder clearUserPayload() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        userPayload_ = getDefaultInstance().getUserPayload();
+        onChanged();
+        return this;
+      }
+
+      // optional int32 version = 2;
+      private int version_ ;
+      /**
+       * <code>optional int32 version = 2;</code>
+       */
+      public boolean hasVersion() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * <code>optional int32 version = 2;</code>
+       */
+      public int getVersion() {
+        return version_;
+      }
+      /**
+       * <code>optional int32 version = 2;</code>
+       */
+      public Builder setVersion(int value) {
+        bitField0_ |= 0x00000002;
+        version_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional int32 version = 2;</code>
+       */
+      public Builder clearVersion() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        version_ = 0;
+        onChanged();
+        return this;
+      }
+
+      // @@protoc_insertion_point(builder_scope:UserPayloadProto)
+    }
+
+    static {
+      defaultInstance = new UserPayloadProto(true);
+      defaultInstance.initFields();
+    }
+
+    // @@protoc_insertion_point(class_scope:UserPayloadProto)
+  }
+
+  public interface EntityDescriptorProtoOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+
+    // optional string class_name = 1;
+    /**
+     * <code>optional string class_name = 1;</code>
+     */
+    boolean hasClassName();
+    /**
+     * <code>optional string class_name = 1;</code>
+     */
+    java.lang.String getClassName();
+    /**
+     * <code>optional string class_name = 1;</code>
+     */
+    com.google.protobuf.ByteString
+        getClassNameBytes();
+
+    // optional .UserPayloadProto user_payload = 2;
+    /**
+     * <code>optional .UserPayloadProto user_payload = 2;</code>
+     */
+    boolean hasUserPayload();
+    /**
+     * <code>optional .UserPayloadProto user_payload = 2;</code>
+     */
+    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload();
+    /**
+     * <code>optional .UserPayloadProto user_payload = 2;</code>
+     */
+    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder();
+
+    // optional bytes history_text = 3;
+    /**
+     * <code>optional bytes history_text = 3;</code>
+     */
+    boolean hasHistoryText();
+    /**
+     * <code>optional bytes history_text = 3;</code>
+     */
+    com.google.protobuf.ByteString getHistoryText();
+  }
+  /**
+   * Protobuf type {@code EntityDescriptorProto}
+   */
+  public static final class EntityDescriptorProto extends
+      com.google.protobuf.GeneratedMessage
+      implements EntityDescriptorProtoOrBuilder {
+    // Use EntityDescriptorProto.newBuilder() to construct.
+    private EntityDescriptorProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+      super(builder);
+      this.unknownFields = builder.getUnknownFields();
+    }
+    private EntityDescriptorProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final EntityDescriptorProto defaultInstance;
+    public static EntityDescriptorProto getDefaultInstance() {
+      return defaultInstance;
+    }
+
+    public EntityDescriptorProto getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @java.lang.Override
+    public final com.google.protobuf.UnknownFieldSet
+        getUnknownFields() {
+      return this.unknownFields;
+    }
+    private EntityDescriptorProto(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      initFields();
+      int mutable_bitField0_ = 0;
+      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder();
+      try {
+        boolean done = false;
+        while (!done) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              done = true;
+              break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              className_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder subBuilder = null;
+              if (((bitField0_ & 0x00000002) == 0x00000002)) {
+                subBuilder = userPayload_.toBuilder();
+              }
+              userPayload_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.PARSER, extensionRegistry);
+              if (subBuilder != null) {
+                subBuilder.mergeFrom(userPayload_);
+                userPayload_ = subBuilder.buildPartial();
+              }
+              bitField0_ |= 0x00000002;
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              historyText_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+        throw e.setUnfinishedMessage(this);
+      } catch (java.io.IOException e) {
+        throw new com.google.protobuf.InvalidProtocolBufferException(
+            e.getMessage()).setUnfinishedMessage(this);
+      } finally {
+        this.unknownFields = unknownFields.build();
+        makeExtensionsImmutable();
+      }
+    }
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
+    }
+
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
+    }
+
+    public static com.google.protobuf.Parser<EntityDescriptorProto> PARSER =
+        new com.google.protobuf.AbstractParser<EntityDescriptorProto>() {
+      public EntityDescriptorProto parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new EntityDescriptorProto(input, extensionRegistry);
+      }
+    };
+
+    @java.lang.Override
+    public com.google.protobuf.Parser<EntityDescriptorProto> getParserForType() {
+      return PARSER;
+    }
+
+    private int bitField0_;
+    // optional string class_name = 1;
+    public static final int CLASS_NAME_FIELD_NUMBER = 1;
+    private java.lang.Object className_;
+    /**
+     * <code>optional string class_name = 1;</code>
+     */
+    public boolean hasClassName() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * <code>optional string class_name = 1;</code>
+     */
+    public java.lang.String getClassName() {
+      java.lang.Object ref = className_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          className_ = s;
+        }
+        return s;
+      }
+    }
+    /**
+     * <code>optional string class_name = 1;</code>
+     */
+    public com.google.protobuf.ByteString
+        getClassNameBytes() {
+      java.lang.Object ref = className_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        className_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    // optional .UserPayloadProto user_payload = 2;
+    public static final int USER_PAYLOAD_FIELD_NUMBER = 2;
+    private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_;
+    /**
+     * <code>optional .UserPayloadProto user_payload = 2;</code>
+     */
+    public boolean hasUserPayload() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * <code>optional .UserPayloadProto user_payload = 2;</code>
+     */
+    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
+      return userPayload_;
+    }
+    /**
+     * <code>optional .UserPayloadProto user_payload = 2;</code>
+     */
+    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
+      return userPayload_;
+    }
+
+    // optional bytes history_text = 3;
+    public static final int HISTORY_TEXT_FIELD_NUMBER = 3;
+    private com.google.protobuf.ByteString historyText_;
+    /**
+     * <code>optional bytes history_text = 3;</code>
+     */
+    public boolean hasHistoryText() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * <code>optional bytes history_text = 3;</code>
+     */
+    public com.google.protobuf.ByteString getHistoryText() {
+      return historyText_;
+    }
+
+    private void initFields() {
+      className_ = "";
+      userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+      historyText_ = com.google.protobuf.ByteString.EMPTY;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getClassNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeMessage(2, userPayload_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, historyText_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getClassNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(2, userPayload_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, historyText_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) obj;
+
+      boolean result = true;
+      result = result && (hasClassName() == other.hasClassName());
+      if (hasClassName()) {
+        result = result && getClassName()
+            .equals(other.getClassName());
+      }
+      result = result && (hasUserPayload() == other.hasUserPayload());
+      if (hasUserPayload()) {
+        result = result && getUserPayload()
+            .equals(other.getUserPayload());
+      }
+      result = result && (hasHistoryText() == other.hasHistoryText());
+      if (hasHistoryText()) {
+        result = result && getHistoryText()
+            .equals(other.getHistoryText());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+
+    private int memoizedHashCode = 0;
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasClassName()) {
+        hash = (37 * hash) + CLASS_NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getClassName().hashCode();
+      }
+      if (hasUserPayload()) {
+        hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
+        hash = (53 * hash) + getUserPayload().hashCode();
+      }
+      if (hasHistoryText()) {
+        hash = (37 * hash) + HISTORY_TEXT_FIELD_NUMBER;
+        hash = (53 * hash) + getHistoryText().hashCode();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return PARSER.parseDelimitedFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input, extensionRegistry);
+    }
+
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /**
+     * Protobuf type {@code EntityDescriptorProto}
+     */
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
+      }
+
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
+      }
+
+      // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+
+      private Builder(
+          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getUserPayloadFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+
+      public Builder clear() {
+        super.clear();
+        className_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        if (userPayloadBuilder_ == null) {
+          userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+        } else {
+          userPayloadBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        historyText_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getDefaultInstanceForType() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto build() {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto buildPartial() {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.className_ = className_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        if (userPayloadBuilder_ == null) {
+          result.userPayload_ = userPayload_;
+        } else {
+          result.userPayload_ = userPayloadBuilder_.build();
+        }
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.historyText_ = historyText_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) {
+          return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other) {
+        if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) return this;
+        if (other.hasClassName()) {
+          bitField0_ |= 0x00000001;
+          className_ = other.className_;
+          onChanged();
+        }
+        if (other.hasUserPayload()) {
+          mergeUserPayload(other.getUserPayload());
+        }
+        if (other.hasHistoryText()) {
+          setHistoryText(other.getHistoryText());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parsedMessage = null;
+        try {
+          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) e.getUnfinishedMessage();
+          throw e;
+        } finally {
+          if (parsedMessage != null) {
+            mergeFrom(parsedMessage);
+          }
+        }
+        return this;
+      }
+      private int bitField0_;
+
+      // optional string class_name = 1;
+      private java.lang.Object className_ = "";
+      /**
+       * <code>optional string class_name = 1;</code>
+       */
+      public boolean hasClassName() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * <code>optional string class_name = 1;</code>
+       */
+      public java.lang.String getClassName() {
+        java.lang.Object ref = className_;
+        if (!(ref instanceof java.lang.String)) {
+          java.lang.String s = ((com.google.protobuf.ByteString) ref)
+              .toStringUtf8();
+          className_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * <code>optional string class_name = 1;</code>
+       */
+      public com.google.protobuf.ByteString
+          getClassNameBytes() {
+        java.lang.Object ref = className_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b = 
+              com.google.protobuf.ByteString.copyFromUtf8(
+                  (java.lang.String) ref);
+          className_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * <code>optional string class_name = 1;</code>
+       */
+      public Builder setClassName(
+          java.lang.String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        className_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional string class_name = 1;</code>
+       */
+      public Builder clearClassName() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        className_ = getDefaultInstance().getClassName();
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional string class_name = 1;</code>
+       */
+      public Builder setClassNameBytes(
+          com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        className_ = value;
+        onChanged();
+        return this;
+      }
+
+      // optional .UserPayloadProto user_payload = 2;
+      private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder> userPayloadBuilder_;
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public boolean hasUserPayload() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
+        if (userPayloadBuilder_ == null) {
+          return userPayload_;
+        } else {
+          return userPayloadBuilder_.getMessage();
+        }
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public Builder setUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
+        if (userPayloadBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          userPayload_ = value;
+          onChanged();
+        } else {
+          userPayloadBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public Builder setUserPayload(
+          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder builderForValue) {
+        if (userPayloadBuilder_ == null) {
+          userPayload_ = builderForValue.build();
+          onChanged();
+        } else {
+          userPayloadBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public Builder mergeUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
+        if (userPayloadBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002) &&
+              userPayload_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) {
+            userPayload_ =
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder(userPayload_).mergeFrom(value).buildPartial();
+          } else {
+            userPayload_ = value;
+          }
+          onChanged();
+        } else {
+          userPayloadBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public Builder clearUserPayload() {
+        if (userPayloadBuilder_ == null) {
+          userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+          onChanged();
+        } else {
+          userPayloadBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder getUserPayloadBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getUserPayloadFieldBuilder().getBuilder();
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
+        if (userPayloadBuilder_ != null) {
+          return userPayloadBuilder_.getMessageOrBuilder();
+        } else {
+          return userPayload_;
+        }
+      }
+      /**
+       * <code>optional .UserPayloadProto user_payload = 2;</code>
+       */
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder> 
+          getUserPayloadFieldBuilder() {
+        if (userPayloadBuilder_ == null) {
+          userPayloadBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder>(
+                  userPayload_,
+                  getParentForChildren(),
+                  isClean());
+          userPayload_ = null;
+        }
+        return userPayloadBuilder_;
+      }
+
+      // optional bytes history_text = 3;
+      private com.google.protobuf.ByteString historyText_ = com.google.protobuf.ByteString.EMPTY;
+      /**
+       * <code>optional bytes history_text = 3;</code>
+       */
+      public boolean hasHistoryText() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      /**
+       * <code>optional bytes history_text = 3;</code>
+       */
+      public com.google.protobuf.ByteString getHistoryText() {
+        return historyText_;
+      }
+      /**
+       * <code>optional bytes history_text = 3;</code>
+       */
+      public Builder setHistoryText(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        historyText_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional bytes history_text = 3;</code>
+       */
+      public Builder clearHistoryText() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        historyText_ = getDefaultInstance().getHistoryText();
+        onChanged();
+        return this;
+      }
+
+      // @@protoc_insertion_point(builder_scope:EntityDescriptorProto)
+    }
+
+    static {
+      defaultInstance = new EntityDescriptorProto(true);
+      defaultInstance.initFields();
+    }
+
+    // @@protoc_insertion_point(class_scope:EntityDescriptorProto)
+  }
+
+  public interface IOSpecProtoOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+
+    // optional string connected_vertex_name = 1;
+    /**
+     * <code>optional string connected_vertex_name = 1;</code>
+     */
+    boolean hasConnectedVertexName();
+    /**
+     * <code>optional string connected_vertex_name = 1;</code>
+     */
+    java.lang.String getConnectedVertexName();
+    /**
+     * <code>optional string connected_vertex_name = 1;</code>
+     */
+    com.google.protobuf.ByteString
+        getConnectedVertexNameBytes();
+
+    // optional .EntityDescriptorProto io_descriptor = 2;
+    /**
+     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+     */
+    boolean hasIoDescriptor();
+    /**
+     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+     */
+    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor();
+    /**
+     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+     */
+    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder();
+
+    // optional int32 physical_edge_count = 3;
+    /**
+     * <code>optional int32 physical_edge_count = 3;</code>
+     */
+    boolean hasPhysicalEdgeCount();
+    /**
+     * <code>optional int32 physical_edge_count = 3;</code>
+     */
+    int getPhysicalEdgeCount();
+  }
+  /**
+   * Protobuf type {@code IOSpecProto}
+   */
+  public static final class IOSpecProto extends
+      com.google.protobuf.GeneratedMessage
+      implements IOSpecProtoOrBuilder {
+    // Use IOSpecProto.newBuilder() to construct.
+    private IOSpecProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+      super(builder);
+      this.unknownFields = builder.getUnknownFields();
+    }
+    private IOSpecProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+    private static final IOSpecProto defaultInstance;
+    public static IOSpecProto getDefaultInstance() {
+      return defaultInstance;
+    }
+
+    public IOSpecProto getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+
+    private final com.google.protobuf.UnknownFieldSet unknownFields;
+    @java.lang.Override
+    public final com.google.protobuf.UnknownFieldSet
+        getUnknownFields() {
+      return this.unknownFields;
+    }
+    private IOSpecProto(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      initFields();
+      int mutable_bitField0_ = 0;
+      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder();
+      try {
+        boolean done = false;
+        while (!done) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              done = true;
+              break;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                done = true;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              connectedVertexName_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder subBuilder = null;
+              if (((bitField0_ & 0x00000002) == 0x00000002)) {
+                subBuilder = ioDescriptor_.toBuilder();
+              }
+              ioDescriptor_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.PARSER, extensionRegistry);
+              if (subBuilder != null) {
+                subBuilder.mergeFrom(ioDescriptor_);
+                ioDescriptor_ = subBuilder.buildPartial();
+              }
+              bitField0_ |= 0x00000002;
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              physicalEdgeCount_ = input.readInt32();
+              break;
+            }
+          }
+        }
+      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+        throw e.setUnfinishedMessage(this);
+      } catch (java.io.IOException e) {
+        throw new com.google.protobuf.InvalidProtocolBufferException(
+            e.getMessage()).setUnfinishedMessage(this);
+      } finally {
+        this.unknownFields = unknownFields.build();
+        makeExtensionsImmutable();
+      }
+    }
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
+    }
+
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
+          .ensureFieldAccessorsInitialized(
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
+    }
+
+    public static com.google.protobuf.Parser<IOSpecProto> PARSER =
+        new com.google.protobuf.AbstractParser<IOSpecProto>() {
+      public IOSpecProto parsePartialFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        return new IOSpecProto(input, extensionRegistry);
+      }
+    };
+
+    @java.lang.Override
+    public com.google.protobuf.Parser<IOSpecProto> getParserForType() {
+      return PARSER;
+    }
+
+    private int bitField0_;
+    // optional string connected_vertex_name = 1;
+    public static final int CONNECTED_VERTEX_NAME_FIELD_NUMBER = 1;
+    private java.lang.Object connectedVertexName_;
+    /**
+     * <code>optional string connected_vertex_name = 1;</code>
+     */
+    public boolean hasConnectedVertexName() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    /**
+     * <code>optional string connected_vertex_name = 1;</code>
+     */
+    public java.lang.String getConnectedVertexName() {
+      java.lang.Object ref = connectedVertexName_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        if (bs.isValidUtf8()) {
+          connectedVertexName_ = s;
+        }
+        return s;
+      }
+    }
+    /**
+     * <code>optional string connected_vertex_name = 1;</code>
+     */
+    public com.google.protobuf.ByteString
+        getConnectedVertexNameBytes() {
+      java.lang.Object ref = connectedVertexName_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        connectedVertexName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    // optional .EntityDescriptorProto io_descriptor = 2;
+    public static final int IO_DESCRIPTOR_FIELD_NUMBER = 2;
+    private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_;
+    /**
+     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+     */
+    public boolean hasIoDescriptor() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    /**
+     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+     */
+    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
+      return ioDescriptor_;
+    }
+    /**
+     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+     */
+    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
+      return ioDescriptor_;
+    }
+
+    // optional int32 physical_edge_count = 3;
+    public static final int PHYSICAL_EDGE_COUNT_FIELD_NUMBER = 3;
+    private int physicalEdgeCount_;
+    /**
+     * <code>optional int32 physical_edge_count = 3;</code>
+     */
+    public boolean hasPhysicalEdgeCount() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    /**
+     * <code>optional int32 physical_edge_count = 3;</code>
+     */
+    public int getPhysicalEdgeCount() {
+      return physicalEdgeCount_;
+    }
+
+    private void initFields() {
+      connectedVertexName_ = "";
+      ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+      physicalEdgeCount_ = 0;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+
+      memoizedIsInitialized = 1;
+      return true;
+    }
+
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getConnectedVertexNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeMessage(2, ioDescriptor_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt32(3, physicalEdgeCount_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getConnectedVertexNameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(2, ioDescriptor_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(3, physicalEdgeCount_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+
+    @java.lang.Override
+    public boolean equals(final java.lang.Object obj) {
+      if (obj == this) {
+       return true;
+      }
+      if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)) {
+        return super.equals(obj);
+      }
+      org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) obj;
+
+      boolean result = true;
+      result = result && (hasConnectedVertexName() == other.hasConnectedVertexName());
+      if (hasConnectedVertexName()) {
+        result = result && getConnectedVertexName()
+            .equals(other.getConnectedVertexName());
+      }
+      result = result && (hasIoDescriptor() == other.hasIoDescriptor());
+      if (hasIoDescriptor()) {
+        result = result && getIoDescriptor()
+            .equals(other.getIoDescriptor());
+      }
+      result = result && (hasPhysicalEdgeCount() == other.hasPhysicalEdgeCount());
+      if (hasPhysicalEdgeCount()) {
+        result = result && (getPhysicalEdgeCount()
+            == other.getPhysicalEdgeCount());
+      }
+      result = result &&
+          getUnknownFields().equals(other.getUnknownFields());
+      return result;
+    }
+
+    private int memoizedHashCode = 0;
+    @java.lang.Override
+    public int hashCode() {
+      if (memoizedHashCode != 0) {
+        return memoizedHashCode;
+      }
+      int hash = 41;
+      hash = (19 * hash) + getDescriptorForType().hashCode();
+      if (hasConnectedVertexName()) {
+        hash = (37 * hash) + CONNECTED_VERTEX_NAME_FIELD_NUMBER;
+        hash = (53 * hash) + getConnectedVertexName().hashCode();
+      }
+      if (hasIoDescriptor()) {
+        hash = (37 * hash) + IO_DESCRIPTOR_FIELD_NUMBER;
+        hash = (53 * hash) + getIoDescriptor().hashCode();
+      }
+      if (hasPhysicalEdgeCount()) {
+        hash = (37 * hash) + PHYSICAL_EDGE_COUNT_FIELD_NUMBER;
+        hash = (53 * hash) + getPhysicalEdgeCount();
+      }
+      hash = (29 * hash) + getUnknownFields().hashCode();
+      memoizedHashCode = hash;
+      return hash;
+    }
+
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return PARSER.parseFrom(data, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return PARSER.parseDelimitedFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseDelimitedFrom(input, extensionRegistry);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input);
+    }
+    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return PARSER.parseFrom(input, extensionRegistry);
+    }
+
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    /**
+     * Protobuf type {@code IOSpecProto}
+     */
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder<Builder>
+       implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProtoOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
+      }
+
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
+            .ensureFieldAccessorsInitialized(
+                org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
+      }
+
+      // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+
+      private Builder(
+          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getIoDescriptorFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+
+      public Builder clear() {
+        super.clear();
+        connectedVertexName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        if (ioDescriptorBuilder_ == null) {
+          ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+        } else {
+          ioDescriptorBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        physicalEdgeCount_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto getDefaultInstanceForType() {
+        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance();
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto build() {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto buildPartial() {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.connectedVertexName_ = connectedVertexName_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        if (ioDescriptorBuilder_ == null) {
+          result.ioDescriptor_ = ioDescriptor_;
+        } else {
+          result.ioDescriptor_ = ioDescriptorBuilder_.build();
+        }
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.physicalEdgeCount_ = physicalEdgeCount_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) {
+          return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+
+      public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other) {
+        if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance()) return this;
+        if (other.hasConnectedVertexName()) {
+          bitField0_ |= 0x00000001;
+          connectedVertexName_ = other.connectedVertexName_;
+          onChanged();
+        }
+        if (other.hasIoDescriptor()) {
+          mergeIoDescriptor(other.getIoDescriptor());
+        }
+        if (other.hasPhysicalEdgeCount()) {
+          setPhysicalEdgeCount(other.getPhysicalEdgeCount());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+
+      public final boolean isInitialized() {
+        return true;
+      }
+
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parsedMessage = null;
+        try {
+          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+          parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) e.getUnfinishedMessage();
+          throw e;
+        } finally {
+          if (parsedMessage != null) {
+            mergeFrom(parsedMessage);
+          }
+        }
+        return this;
+      }
+      private int bitField0_;
+
+      // optional string connected_vertex_name = 1;
+      private java.lang.Object connectedVertexName_ = "";
+      /**
+       * <code>optional string connected_vertex_name = 1;</code>
+       */
+      public boolean hasConnectedVertexName() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      /**
+       * <code>optional string connected_vertex_name = 1;</code>
+       */
+      public java.lang.String getConnectedVertexName() {
+        java.lang.Object ref = connectedVertexName_;
+        if (!(ref instanceof java.lang.String)) {
+          java.lang.String s = ((com.google.protobuf.ByteString) ref)
+              .toStringUtf8();
+          connectedVertexName_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * <code>optional string connected_vertex_name = 1;</code>
+       */
+      public com.google.protobuf.ByteString
+          getConnectedVertexNameBytes() {
+        java.lang.Object ref = connectedVertexName_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b = 
+              com.google.protobuf.ByteString.copyFromUtf8(
+                  (java.lang.String) ref);
+          connectedVertexName_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * <code>optional string connected_vertex_name = 1;</code>
+       */
+      public Builder setConnectedVertexName(
+          java.lang.String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        connectedVertexName_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional string connected_vertex_name = 1;</code>
+       */
+      public Builder clearConnectedVertexName() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        connectedVertexName_ = getDefaultInstance().getConnectedVertexName();
+        onChanged();
+        return this;
+      }
+      /**
+       * <code>optional string connected_vertex_name = 1;</code>
+       */
+      public Builder setConnectedVertexNameBytes(
+          com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        connectedVertexName_ = value;
+        onChanged();
+        return this;
+      }
+
+      // optional .EntityDescriptorProto io_descriptor = 2;
+      private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder> ioDescriptorBuilder_;
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public boolean hasIoDescriptor() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
+        if (ioDescriptorBuilder_ == null) {
+          return ioDescriptor_;
+        } else {
+          return ioDescriptorBuilder_.getMessage();
+        }
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public Builder setIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
+        if (ioDescriptorBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ioDescriptor_ = value;
+          onChanged();
+        } else {
+          ioDescriptorBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public Builder setIoDescriptor(
+          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder builderForValue) {
+        if (ioDescriptorBuilder_ == null) {
+          ioDescriptor_ = builderForValue.build();
+          onChanged();
+        } else {
+          ioDescriptorBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public Builder mergeIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
+        if (ioDescriptorBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002) &&
+              ioDescriptor_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) {
+            ioDescriptor_ =
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder(ioDescriptor_).mergeFrom(value).buildPartial();
+          } else {
+            ioDescriptor_ = value;
+          }
+          onChanged();
+        } else {
+          ioDescriptorBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public Builder clearIoDescriptor() {
+        if (ioDescriptorBuilder_ == null) {
+          ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+          onChanged();
+        } else {
+          ioDescriptorBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder getIoDescriptorBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getIoDescriptorFieldBuilder().getBuilder();
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
+        if (ioDescriptorBuilder_ != null) {
+          return ioDescriptorBuilder_.getMessageOrBuilder();
+        } else {
+          return ioDescriptor_;
+        }
+      }
+      /**
+       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+       */
+      private com.google.protobuf.SingleFieldBuilder<
+          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder> 
+          getIoDescriptorFieldBuilder() {
+        if (ioDescriptorBuilder_ == null) {
+          ioDescriptorBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder>(
+                  ioDescriptor_,
+                  getParentForChildren(),
+                  isClean());
+          ioDescriptor_ = null;
+        }
+        return ioDescriptorBuilder_;
+      }
+
+      // optional int32 physical_edge_count = 3;
+      private int physicalEdgeCount_ ;
+      /**
+       * <code>optional int32 physical_edge_cou

<TRUNCATED>

[10/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin)
(cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9886414b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9886414b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9886414b

Branch: refs/heads/branch-2.0
Commit: 9886414b3dd973c1630b84634f0296adac7a193b
Parents: 29a988f
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 1 09:45:09 2016 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Feb 1 09:46:18 2016 -0800

----------------------------------------------------------------------
 llap-client/pom.xml                             |    87 +-
 .../hive/llap/security/LlapTokenIdentifier.java |    82 -
 .../hive/llap/security/LlapTokenProvider.java   |    27 -
 .../hive/llap/tez/LlapProtocolClientProxy.java  |   509 +
 .../tez/TestLlapDaemonProtocolClientProxy.java  |   144 +
 llap-common/pom.xml                             |   235 +
 .../daemon/rpc/LlapDaemonProtocolProtos.java    | 14553 +++++++++++++++++
 .../org/apache/hadoop/hive/llap/LlapNodeId.java |    86 +
 .../impl/LlapManagementProtocolClientImpl.java  |    82 +
 .../hive/llap/impl/LlapProtocolClientImpl.java  |   125 +
 .../llap/protocol/LlapManagementProtocolPB.java |    26 +
 .../llap/protocol/LlapProtocolBlockingPB.java   |    30 +
 .../protocol/LlapTaskUmbilicalProtocol.java     |    42 +
 .../hive/llap/security/LlapTokenIdentifier.java |    82 +
 .../hive/llap/security/LlapTokenProvider.java   |    27 +
 .../hive/llap/security/LlapTokenSelector.java   |    53 +
 .../apache/hadoop/hive/llap/tez/Converters.java |   265 +
 .../src/protobuf/LlapDaemonProtocol.proto       |   148 +
 .../hive/llap/testhelpers/ControlledClock.java  |    43 +
 llap-server/pom.xml                             |   199 +-
 .../daemon/rpc/LlapDaemonProtocolProtos.java    | 14553 -----------------
 .../org/apache/hadoop/hive/llap/LlapNodeId.java |    86 -
 .../hadoop/hive/llap/cli/LlapServiceDriver.java |     8 +
 .../daemon/LlapDaemonProtocolBlockingPB.java    |    28 -
 .../LlapManagementProtocolBlockingPB.java       |    24 -
 .../hive/llap/daemon/impl/LlapDaemon.java       |     4 +-
 .../impl/LlapDaemonProtocolClientImpl.java      |   125 -
 .../impl/LlapDaemonProtocolServerImpl.java      |   251 -
 .../impl/LlapManagementProtocolClientImpl.java  |    82 -
 .../daemon/impl/LlapProtocolServerImpl.java     |   251 +
 .../llap/daemon/impl/QueryFragmentInfo.java     |    13 +-
 .../llap/daemon/impl/TaskRunnerCallable.java    |     2 +-
 .../protocol/LlapTaskUmbilicalProtocol.java     |    42 -
 .../llap/security/LlapDaemonPolicyProvider.java |     8 +-
 .../hive/llap/security/LlapSecurityHelper.java  |     2 +-
 .../llap/security/LlapServerSecurityInfo.java   |    10 +-
 .../hive/llap/security/LlapTokenSelector.java   |    53 -
 .../hadoop/hive/llap/tezplugins/Converters.java |   265 -
 .../llap/tezplugins/LlapContainerLauncher.java  |    43 -
 .../LlapDaemonProtocolClientProxy.java          |   509 -
 .../llap/tezplugins/LlapTaskCommunicator.java   |   755 -
 .../tezplugins/LlapUmbilicalPolicyProvider.java |    42 -
 .../tezplugins/helpers/SourceStateTracker.java  |   296 -
 .../apache/tez/dag/app/rm/ContainerFactory.java |    51 -
 .../dag/app/rm/LlapTaskSchedulerService.java    |  1512 --
 .../src/protobuf/LlapDaemonProtocol.proto       |   148 -
 .../impl/TestLlapDaemonProtocolServerImpl.java  |    11 +-
 .../TestLlapDaemonProtocolClientProxy.java      |   143 -
 .../tezplugins/TestLlapTaskCommunicator.java    |   100 -
 .../app/rm/TestLlapTaskSchedulerService.java    |   685 -
 llap-tez/pom.xml                                |   200 +
 .../hive/llap/tezplugins/ContainerFactory.java  |    51 +
 .../llap/tezplugins/LlapContainerLauncher.java  |    43 +
 .../llap/tezplugins/LlapTaskCommunicator.java   |   757 +
 .../tezplugins/LlapTaskSchedulerService.java    |  1512 ++
 .../hive/llap/tezplugins/LlapTezUtils.java      |    29 +
 .../tezplugins/LlapUmbilicalPolicyProvider.java |    42 +
 .../tezplugins/helpers/SourceStateTracker.java  |   289 +
 .../tezplugins/TestLlapTaskCommunicator.java    |   100 +
 .../TestLlapTaskSchedulerService.java           |   684 +
 pom.xml                                         |     4 +-
 ql/pom.xml                                      |    54 +
 .../hive/ql/exec/tez/TezSessionState.java       |    58 +-
 63 files changed, 20631 insertions(+), 20139 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/pom.xml
----------------------------------------------------------------------
diff --git a/llap-client/pom.xml b/llap-client/pom.xml
index 3c85180..a0eeb44 100644
--- a/llap-client/pom.xml
+++ b/llap-client/pom.xml
@@ -32,7 +32,7 @@
   </properties>
 
   <dependencies>
-    <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+    <!-- dependencies are always listed in sorted order by groupId, artifactId -->
     <!-- intra-project -->
     <dependency>
       <groupId>org.apache.hive</groupId>
@@ -41,41 +41,26 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
-      <artifactId>hive-serde</artifactId>
+      <artifactId>hive-llap-common</artifactId>
       <version>${project.version}</version>
     </dependency>
     <!-- inter-project -->
     <dependency>
-      <groupId>commons-codec</groupId>
-      <artifactId>commons-codec</artifactId>
-      <version>${commons-codec.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>commons-lang</groupId>
-      <artifactId>commons-lang</artifactId>
-      <version>${commons-lang.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.thrift</groupId>
-      <artifactId>libthrift</artifactId>
-      <version>${libthrift.version}</version>
-    </dependency>
-    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>
       <optional>true</optional>
-          <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
-   </dependency>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-core</artifactId>
@@ -96,15 +81,15 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <version>${mockito-all.version}</version>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>${commons-lang3.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-servlet</artifactId>
-      <version>${jersey.version}</version>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>${mockito-all.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -113,32 +98,18 @@
       <version>${hadoop.version}</version>
       <classifier>tests</classifier>
       <scope>test</scope>
-          <exclusions>
-            <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>commmons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-          </exclusion>
-        </exclusions>
-   </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop.version}</version>
-      <classifier>tests</classifier>
-      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
-
   <build>
     <sourceDirectory>${basedir}/src/java</sourceDirectory>
     <testSourceDirectory>${basedir}/src/test</testSourceDirectory>

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
deleted file mode 100644
index f0bb495..0000000
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-
-/** For now, a LLAP token gives access to any LLAP server. */
-public class LlapTokenIdentifier extends AbstractDelegationTokenIdentifier {
-  private static final String KIND = "LLAP_TOKEN";
-  public static final Text KIND_NAME = new Text(KIND);
-
-  public LlapTokenIdentifier() {
-    super();
-  }
-
-  public LlapTokenIdentifier(Text owner, Text renewer, Text realUser) {
-    super(owner, renewer, realUser);
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    // Nothing right now.
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    // Nothing right now.
-  }
-
-  @Override
-  public Text getKind() {
-    return KIND_NAME;
-  }
-
-  @Override
-  public int hashCode() {
-    return -1;
-  }
-
-  @Override
-  public boolean equals(Object other) {
-    return (other != null) && other.getClass().isAssignableFrom(this.getClass());
-  }
-
-  @Override
-  public String toString() {
-    return KIND;
-  }
-
-  @InterfaceAudience.Private
-  public static class Renewer extends Token.TrivialRenewer {
-    @Override
-    protected Text getKind() {
-      return KIND_NAME;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
deleted file mode 100644
index 2e99a28..0000000
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import java.io.IOException;
-
-import org.apache.hadoop.security.token.Token;
-
-public interface LlapTokenProvider {
-  Token<LlapTokenIdentifier> getDelegationToken() throws IOException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
new file mode 100644
index 0000000..5b0674a
--- /dev/null
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
@@ -0,0 +1,509 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import javax.net.SocketFactory;
+
+import java.io.IOException;
+import java.security.PrivilegedAction;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.Message;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.impl.LlapProtocolClientImpl;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapProtocolClientProxy extends AbstractService {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LlapProtocolClientProxy.class);
+
+  private final ConcurrentMap<String, LlapProtocolBlockingPB> hostProxies;
+
+  private final RequestManager requestManager;
+  private final RetryPolicy retryPolicy;
+  private final SocketFactory socketFactory;
+
+  private final ListeningExecutorService requestManagerExecutor;
+  private volatile ListenableFuture<Void> requestManagerFuture;
+  private final Token<LlapTokenIdentifier> llapToken;
+
+  public LlapProtocolClientProxy(
+      int numThreads, Configuration conf, Token<LlapTokenIdentifier> llapToken) {
+    super(LlapProtocolClientProxy.class.getSimpleName());
+    this.hostProxies = new ConcurrentHashMap<>();
+    this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+    this.llapToken = llapToken;
+
+    long connectionTimeout = HiveConf.getTimeVar(conf,
+        ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+    long retrySleep = HiveConf.getTimeVar(conf,
+        ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MS,
+        TimeUnit.MILLISECONDS);
+    this.retryPolicy = RetryPolicies.retryUpToMaximumTimeWithFixedSleep(
+        connectionTimeout, retrySleep, TimeUnit.MILLISECONDS);
+
+    this.requestManager = new RequestManager(numThreads);
+    ExecutorService localExecutor = Executors.newFixedThreadPool(1,
+        new ThreadFactoryBuilder().setNameFormat("RequestManagerExecutor").build());
+    this.requestManagerExecutor = MoreExecutors.listeningDecorator(localExecutor);
+
+    LOG.info("Setting up taskCommunicator with" +
+        "numThreads=" + numThreads +
+        "retryTime(millis)=" + connectionTimeout +
+        "retrySleep(millis)=" + retrySleep);
+  }
+
+  @Override
+  public void serviceStart() {
+    requestManagerFuture = requestManagerExecutor.submit(requestManager);
+    Futures.addCallback(requestManagerFuture, new FutureCallback<Void>() {
+      @Override
+      public void onSuccess(Void result) {
+        LOG.info("RequestManager shutdown");
+      }
+
+      @Override
+      public void onFailure(Throwable t) {
+        LOG.warn("RequestManager shutdown with error", t);
+      }
+    });
+  }
+
+  @Override
+  public void serviceStop() {
+    if (requestManagerFuture != null) {
+      requestManager.shutdown();
+      requestManagerFuture.cancel(true);
+    }
+    requestManagerExecutor.shutdown();
+  }
+
+  public void sendSubmitWork(SubmitWorkRequestProto request, String host, int port,
+                         final ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
+    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+    requestManager.queueRequest(new SubmitWorkCallable(nodeId, request, callback));
+  }
+
+  public void sendSourceStateUpdate(final SourceStateUpdatedRequestProto request, final String host,
+                                    final int port,
+                                    final ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
+    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+    requestManager.queueRequest(
+        new SendSourceStateUpdateCallable(nodeId, request, callback));
+  }
+
+  public void sendQueryComplete(final QueryCompleteRequestProto request, final String host,
+                                final int port,
+                                final ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
+    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+    requestManager.queueRequest(new SendQueryCompleteCallable(nodeId, request, callback));
+  }
+
+  public void sendTerminateFragment(final TerminateFragmentRequestProto request, final String host,
+                                    final int port,
+                                    final ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
+    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+    requestManager.queueRequest(new SendTerminateFragmentCallable(nodeId, request, callback));
+  }
+
+  @VisibleForTesting
+  static class RequestManager implements Callable<Void> {
+
+    private final Lock lock = new ReentrantLock();
+    private final AtomicBoolean isShutdown = new AtomicBoolean(false);
+    private final Condition queueCondition = lock.newCondition();
+    private final AtomicBoolean shouldRun = new AtomicBoolean(false);
+
+    private final int maxConcurrentRequestsPerNode = 1;
+    private final ListeningExecutorService executor;
+
+
+    // Tracks new additions via add, while the loop is processing existing ones.
+    private final LinkedList<CallableRequest> newRequestList = new LinkedList<>();
+
+    // Tracks existing requests which are cycled through.
+    private final LinkedList<CallableRequest> pendingRequests = new LinkedList<>();
+
+    // Tracks requests executing per node
+    private final ConcurrentMap<LlapNodeId, AtomicInteger> runningRequests = new ConcurrentHashMap<>();
+
+    // Tracks completed requests pre node
+    private final LinkedList<LlapNodeId> completedNodes = new LinkedList<>();
+
+    public RequestManager(int numThreads) {
+      ExecutorService localExecutor = Executors.newFixedThreadPool(numThreads,
+          new ThreadFactoryBuilder().setNameFormat("TaskCommunicator #%2d").build());
+      executor = MoreExecutors.listeningDecorator(localExecutor);
+    }
+
+
+    @VisibleForTesting
+    Set<LlapNodeId> currentLoopDisabledNodes = new HashSet<>();
+    @VisibleForTesting
+    List<CallableRequest> currentLoopSkippedRequests = new LinkedList<>();
+    @Override
+    public Void call() {
+      // Caches disabled nodes for quicker lookups and ensures a request on a node which was skipped
+      // does not go out of order.
+      while (!isShutdown.get()) {
+        lock.lock();
+        try {
+          while (!shouldRun.get()) {
+            queueCondition.await();
+            break; // Break out and try executing.
+          }
+          boolean shouldBreak = process();
+          if (shouldBreak) {
+            break;
+          }
+        } catch (InterruptedException e) {
+          if (isShutdown.get()) {
+            break;
+          } else {
+            LOG.warn("RunLoop interrupted without being shutdown first");
+            throw new RuntimeException(e);
+          }
+        } finally {
+          lock.unlock();
+        }
+      }
+      LOG.info("CallScheduler loop exiting");
+      return null;
+    }
+
+    /* Add a new request to be executed */
+    public void queueRequest(CallableRequest request) {
+      synchronized (newRequestList) {
+        newRequestList.add(request);
+        shouldRun.set(true);
+      }
+      notifyRunLoop();
+    }
+
+    /* Indicates a request has completed on a node */
+    public void requestFinished(LlapNodeId nodeId) {
+      synchronized (completedNodes) {
+        completedNodes.add(nodeId);
+        shouldRun.set(true);
+      }
+      notifyRunLoop();
+    }
+
+    public void shutdown() {
+      if (!isShutdown.getAndSet(true)) {
+        executor.shutdownNow();
+        notifyRunLoop();
+      }
+    }
+
+    @VisibleForTesting
+    void submitToExecutor(CallableRequest request, LlapNodeId nodeId) {
+      ListenableFuture<SourceStateUpdatedResponseProto> future =
+          executor.submit(request);
+      Futures.addCallback(future, new ResponseCallback(request.getCallback(), nodeId, this));
+    }
+
+    @VisibleForTesting
+    boolean process() {
+      if (isShutdown.get()) {
+        return true;
+      }
+      currentLoopDisabledNodes.clear();
+      currentLoopSkippedRequests.clear();
+
+      // Set to false to block the next loop. This must be called before draining the lists,
+      // otherwise an add/completion after draining the lists but before setting it to false,
+      // will not trigger a run. May cause one unnecessary run if an add comes in before drain.
+      // drain list. add request (setTrue). setFalse needs to be avoided.
+      shouldRun.compareAndSet(true, false);
+      // Drain any calls which may have come in during the last execution of the loop.
+      drainNewRequestList();  // Locks newRequestList
+      drainCompletedNodes();  // Locks completedNodes
+
+
+      Iterator<CallableRequest> iterator = pendingRequests.iterator();
+      while (iterator.hasNext()) {
+        CallableRequest request = iterator.next();
+        iterator.remove();
+        LlapNodeId nodeId = request.getNodeId();
+        if (canRunForNode(nodeId, currentLoopDisabledNodes)) {
+          submitToExecutor(request, nodeId);
+        } else {
+          currentLoopDisabledNodes.add(nodeId);
+          currentLoopSkippedRequests.add(request);
+        }
+      }
+      // Tried scheduling everything that could be scheduled in this loop.
+      pendingRequests.addAll(0, currentLoopSkippedRequests);
+      return false;
+    }
+
+    private void drainNewRequestList() {
+      synchronized (newRequestList) {
+        if (!newRequestList.isEmpty()) {
+          pendingRequests.addAll(newRequestList);
+          newRequestList.clear();
+        }
+      }
+    }
+
+    private void drainCompletedNodes() {
+      synchronized (completedNodes) {
+        if (!completedNodes.isEmpty()) {
+          for (LlapNodeId nodeId : completedNodes) {
+            runningRequests.get(nodeId).decrementAndGet();
+          }
+        }
+        completedNodes.clear();
+      }
+    }
+
+    private boolean canRunForNode(LlapNodeId nodeId, Set<LlapNodeId> currentRunDisabledNodes) {
+      if (currentRunDisabledNodes.contains(nodeId)) {
+        return false;
+      } else {
+        AtomicInteger count = runningRequests.get(nodeId);
+        if (count == null) {
+          count = new AtomicInteger(0);
+          AtomicInteger old = runningRequests.putIfAbsent(nodeId, count);
+          count = old != null ? old : count;
+        }
+        if (count.incrementAndGet() <= maxConcurrentRequestsPerNode) {
+          return true;
+        } else {
+          count.decrementAndGet();
+          return false;
+        }
+      }
+    }
+
+    private void notifyRunLoop() {
+      lock.lock();
+      try {
+        queueCondition.signal();
+      } finally {
+        lock.unlock();
+      }
+    }
+  }
+
+
+  private static final class ResponseCallback<TYPE extends Message>
+      implements FutureCallback<TYPE> {
+
+    private final ExecuteRequestCallback<TYPE> callback;
+    private final LlapNodeId nodeId;
+    private final RequestManager requestManager;
+
+    public ResponseCallback(ExecuteRequestCallback<TYPE> callback, LlapNodeId nodeId,
+                            RequestManager requestManager) {
+      this.callback = callback;
+      this.nodeId = nodeId;
+      this.requestManager = requestManager;
+    }
+
+    @Override
+    public void onSuccess(TYPE result) {
+      try {
+        callback.setResponse(result);
+      } finally {
+        requestManager.requestFinished(nodeId);
+      }
+    }
+
+    @Override
+    public void onFailure(Throwable t) {
+      try {
+        callback.indicateError(t);
+      } finally {
+        requestManager.requestFinished(nodeId);
+      }
+    }
+  }
+
+  @VisibleForTesting
+  static abstract class CallableRequest<REQUEST extends Message, RESPONSE extends Message>
+      implements Callable {
+
+    final LlapNodeId nodeId;
+    final ExecuteRequestCallback<RESPONSE> callback;
+    final REQUEST request;
+
+
+    protected CallableRequest(LlapNodeId nodeId, REQUEST request, ExecuteRequestCallback<RESPONSE> callback) {
+      this.nodeId = nodeId;
+      this.request = request;
+      this.callback = callback;
+    }
+
+    public LlapNodeId getNodeId() {
+      return nodeId;
+    }
+
+    public ExecuteRequestCallback<RESPONSE> getCallback() {
+      return callback;
+    }
+
+    public abstract RESPONSE call() throws Exception;
+  }
+
+  private class SubmitWorkCallable extends CallableRequest<SubmitWorkRequestProto, SubmitWorkResponseProto> {
+
+    protected SubmitWorkCallable(LlapNodeId nodeId,
+                          SubmitWorkRequestProto submitWorkRequestProto,
+                                 ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
+      super(nodeId, submitWorkRequestProto, callback);
+    }
+
+    @Override
+    public SubmitWorkResponseProto call() throws Exception {
+      return getProxy(nodeId).submitWork(null, request);
+    }
+  }
+
+  private class SendSourceStateUpdateCallable
+      extends CallableRequest<SourceStateUpdatedRequestProto, SourceStateUpdatedResponseProto> {
+
+    public SendSourceStateUpdateCallable(LlapNodeId nodeId,
+                                         SourceStateUpdatedRequestProto request,
+                                         ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
+      super(nodeId, request, callback);
+    }
+
+    @Override
+    public SourceStateUpdatedResponseProto call() throws Exception {
+      return getProxy(nodeId).sourceStateUpdated(null, request);
+    }
+  }
+
+  private class SendQueryCompleteCallable
+      extends CallableRequest<QueryCompleteRequestProto, QueryCompleteResponseProto> {
+
+    protected SendQueryCompleteCallable(LlapNodeId nodeId,
+                                        QueryCompleteRequestProto queryCompleteRequestProto,
+                                        ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
+      super(nodeId, queryCompleteRequestProto, callback);
+    }
+
+    @Override
+    public QueryCompleteResponseProto call() throws Exception {
+      return getProxy(nodeId).queryComplete(null, request);
+    }
+  }
+
+  private class SendTerminateFragmentCallable
+      extends CallableRequest<TerminateFragmentRequestProto, TerminateFragmentResponseProto> {
+
+    protected SendTerminateFragmentCallable(LlapNodeId nodeId,
+                                            TerminateFragmentRequestProto terminateFragmentRequestProto,
+                                            ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
+      super(nodeId, terminateFragmentRequestProto, callback);
+    }
+
+    @Override
+    public TerminateFragmentResponseProto call() throws Exception {
+      return getProxy(nodeId).terminateFragment(null, request);
+    }
+  }
+
+  public interface ExecuteRequestCallback<T extends Message> {
+    void setResponse(T response);
+    void indicateError(Throwable t);
+  }
+
+  private LlapProtocolBlockingPB getProxy(final LlapNodeId nodeId) {
+    String hostId = getHostIdentifier(nodeId.getHostname(), nodeId.getPort());
+
+    LlapProtocolBlockingPB proxy = hostProxies.get(hostId);
+    if (proxy == null) {
+      if (llapToken == null) {
+        proxy = new LlapProtocolClientImpl(getConfig(), nodeId.getHostname(),
+            nodeId.getPort(), retryPolicy, socketFactory);
+      } else {
+        UserGroupInformation ugi;
+        try {
+          ugi = UserGroupInformation.getCurrentUser();
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+        Token<LlapTokenIdentifier> nodeToken = new Token<LlapTokenIdentifier>(llapToken);
+        SecurityUtil.setTokenService(nodeToken, NetUtils.createSocketAddrForHost(
+            nodeId.getHostname(), nodeId.getPort()));
+        ugi.addToken(nodeToken);
+        proxy = ugi.doAs(new PrivilegedAction<LlapProtocolBlockingPB>() {
+          @Override
+          public LlapProtocolBlockingPB run() {
+           return new LlapProtocolClientImpl(getConfig(), nodeId.getHostname(),
+               nodeId.getPort(), retryPolicy, socketFactory);
+          }
+        });
+      }
+
+      LlapProtocolBlockingPB proxyOld = hostProxies.putIfAbsent(hostId, proxy);
+      if (proxyOld != null) {
+        // TODO Shutdown the new proxy.
+        proxy = proxyOld;
+      }
+    }
+    return proxy;
+  }
+
+  private String getHostIdentifier(String hostname, int port) {
+    return hostname + ":" + port;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java b/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java
new file mode 100644
index 0000000..850b67a
--- /dev/null
+++ b/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.protobuf.Message;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestLlapDaemonProtocolClientProxy {
+
+  @Test (timeout = 5000)
+  public void testMultipleNodes() {
+    RequestManagerForTest requestManager = new RequestManagerForTest(1);
+
+    LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
+    LlapNodeId nodeId2 = LlapNodeId.getInstance("host2", 1025);
+
+    Message mockMessage = mock(Message.class);
+    LlapProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
+        LlapProtocolClientProxy.ExecuteRequestCallback.class);
+
+    // Request two messages
+    requestManager.queueRequest(
+        new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
+    requestManager.queueRequest(
+        new CallableRequestForTest(nodeId2, mockMessage, mockExecuteRequestCallback));
+
+    // Should go through in a single process call
+    requestManager.process();
+    assertEquals(2, requestManager.numSubmissionsCounters);
+    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId2));
+    Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+    Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId2).getValue().intValue());
+    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+    assertEquals(0, requestManager.currentLoopDisabledNodes.size());
+  }
+
+  @Test(timeout = 5000)
+  public void testSingleInvocationPerNode() {
+    RequestManagerForTest requestManager = new RequestManagerForTest(1);
+
+    LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
+
+    Message mockMessage = mock(Message.class);
+    LlapProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
+        LlapProtocolClientProxy.ExecuteRequestCallback.class);
+
+    // First request for host.
+    requestManager.queueRequest(
+        new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
+    requestManager.process();
+    assertEquals(1, requestManager.numSubmissionsCounters);
+    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+    Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+
+    // Second request for host. Single invocation since the last has not completed.
+    requestManager.queueRequest(
+        new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
+    requestManager.process();
+    assertEquals(1, requestManager.numSubmissionsCounters);
+    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+    Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+    assertEquals(1, requestManager.currentLoopSkippedRequests.size());
+    assertEquals(1, requestManager.currentLoopDisabledNodes.size());
+    assertTrue(requestManager.currentLoopDisabledNodes.contains(nodeId1));
+
+    // Complete first request. Second pending request should go through.
+    requestManager.requestFinished(nodeId1);
+    requestManager.process();
+    assertEquals(2, requestManager.numSubmissionsCounters);
+    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+    Assert.assertEquals(2, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+    assertEquals(0, requestManager.currentLoopDisabledNodes.size());
+    assertFalse(requestManager.currentLoopDisabledNodes.contains(nodeId1));
+  }
+
+
+  static class RequestManagerForTest extends LlapProtocolClientProxy.RequestManager {
+
+    int numSubmissionsCounters = 0;
+    private Map<LlapNodeId, MutableInt> numInvocationsPerNode = new HashMap<>();
+
+    public RequestManagerForTest(int numThreads) {
+      super(numThreads);
+    }
+
+    protected void submitToExecutor(LlapProtocolClientProxy.CallableRequest request, LlapNodeId nodeId) {
+      numSubmissionsCounters++;
+      MutableInt nodeCount = numInvocationsPerNode.get(nodeId);
+      if (nodeCount == null) {
+        nodeCount = new MutableInt(0);
+        numInvocationsPerNode.put(nodeId, nodeCount);
+      }
+      nodeCount.increment();
+    }
+
+    void reset() {
+      numSubmissionsCounters = 0;
+      numInvocationsPerNode.clear();
+    }
+
+  }
+
+  static class CallableRequestForTest extends LlapProtocolClientProxy.CallableRequest<Message, Message> {
+
+    protected CallableRequestForTest(LlapNodeId nodeId, Message message,
+                                     LlapProtocolClientProxy.ExecuteRequestCallback<Message> callback) {
+      super(nodeId, message, callback);
+    }
+
+    @Override
+    public Message call() throws Exception {
+      return null;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/pom.xml
----------------------------------------------------------------------
diff --git a/llap-common/pom.xml b/llap-common/pom.xml
new file mode 100644
index 0000000..5343479
--- /dev/null
+++ b/llap-common/pom.xml
@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>hive-llap-common</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Llap Common</name>
+
+  <properties>
+    <hive.path.to.root>..</hive.path.to.root>
+  </properties>
+
+  <dependencies>
+    <!-- dependencies are always listed in sorted order by groupId, artifactId -->
+    <!-- intra-project -->
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <!-- inter-project -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <optional>true</optional>
+            <exclusions>
+            <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>commmons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+        </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-api</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-common</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-runtime-internals</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- test inter-project -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>${mockito-all.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <profiles>
+    <profile>
+      <id>protobuf</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>generate-protobuf-sources</id>
+                <phase>generate-sources</phase>
+                <configuration>
+                  <target>
+                    <property name="protobuf.src.dir"  location="${basedir}/src/protobuf"/>
+                    <property name="protobuf.build.dir"  location="${basedir}/src/gen/protobuf/gen-java"/>
+                    <echo>Building LLAP Protobuf</echo>
+                    <mkdir dir="${protobuf.build.dir}"/>
+                    <exec executable="protoc" failonerror="true">
+                      <arg value="--java_out=${protobuf.build.dir}"/>
+                      <arg value="-I=${protobuf.src.dir}"/>
+                      <arg value="${protobuf.src.dir}/LlapDaemonProtocol.proto"/>
+                    </exec>
+                  </target>
+                </configuration>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-jar-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>test-jar</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+
+
+  <build>
+    <sourceDirectory>${basedir}/src/java</sourceDirectory>
+    <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+    <resources>
+    <resource>
+      <directory>src/main/resources</directory>
+      <excludes>
+        <exclude>*.py</exclude>
+        <exclude>*.pyc</exclude>
+      </excludes>
+      <filtering>false</filtering>
+    </resource>
+    </resources>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/gen/protobuf/gen-java</source>
+                <source>src/gen/thrift/gen-javabean</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>


[03/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java b/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
deleted file mode 100644
index 4c1cbb3..0000000
--- a/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
+++ /dev/null
@@ -1,685 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
-import org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.ControlledClock;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestLlapTaskSchedulerService {
-
-  private static final Logger LOG = LoggerFactory.getLogger(TestLlapTaskSchedulerService.class);
-
-  private static final String HOST1 = "host1";
-  private static final String HOST2 = "host2";
-  private static final String HOST3 = "host3";
-
-  @Test (timeout = 5000)
-  public void testSimpleLocalAllocation() throws IOException, InterruptedException {
-
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
-
-    try {
-      Priority priority1 = Priority.newInstance(1);
-      String[] hosts1 = new String[]{HOST1};
-
-      Object task1 = new Object();
-      Object clientCookie1 = new Object();
-
-      tsWrapper.controlScheduler(true);
-      tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
-
-      tsWrapper.signalSchedulerRun();
-      tsWrapper.awaitSchedulerRun();
-
-      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
-      // TODO Verify this is on host1.
-      assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-  @Test (timeout = 5000)
-  public void testSimpleNoLocalityAllocation() throws IOException, InterruptedException {
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
-
-    try {
-      Priority priority1 = Priority.newInstance(1);
-
-      Object task1 = new Object();
-      Object clientCookie1 = new Object();
-      tsWrapper.controlScheduler(true);
-      tsWrapper.allocateTask(task1, null, priority1, clientCookie1);
-      tsWrapper.signalSchedulerRun();
-      tsWrapper.awaitSchedulerRun();
-      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
-      assertEquals(1, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-
-  @Test(timeout=5000)
-  public void testPreemption() throws InterruptedException, IOException {
-
-    Priority priority1 = Priority.newInstance(1);
-    Priority priority2 = Priority.newInstance(2);
-    String [] hosts = new String[] {HOST1};
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1);
-    try {
-
-      Object task1 = "task1";
-      Object clientCookie1 = "cookie1";
-      Object task2 = "task2";
-      Object clientCookie2 = "cookie2";
-      Object task3 = "task3";
-      Object clientCookie3 = "cookie3";
-      Object task4 = "task4";
-      Object clientCookie4 = "cookie4";
-
-      tsWrapper.controlScheduler(true);
-      tsWrapper.allocateTask(task1, hosts, priority2, clientCookie1);
-      tsWrapper.allocateTask(task2, hosts, priority2, clientCookie2);
-      tsWrapper.allocateTask(task3, hosts, priority2, clientCookie3);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(any(Object.class),
-          any(Object.class), any(Container.class));
-      assertEquals(2, tsWrapper.ts.dagStats.numLocalAllocations);
-      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-
-      reset(tsWrapper.mockAppCallback);
-
-      tsWrapper.allocateTask(task4, hosts, priority1, clientCookie4);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
-
-
-      tsWrapper.deallocateTask(task2, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
-
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
-          eq(clientCookie4), any(Container.class));
-
-    } finally {
-      tsWrapper.shutdown();
-    }
-
-  }
-
-  @Test(timeout=5000)
-  public void testNodeDisabled() throws IOException, InterruptedException {
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(10000l);
-    try {
-      Priority priority1 = Priority.newInstance(1);
-      String[] hosts1 = new String[]{HOST1};
-      Object task1 = new Object();
-      Object clientCookie1 = new Object();
-      tsWrapper.controlScheduler(true);
-      tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 1) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1),
-          any(Container.class));
-      assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
-      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-      assertEquals(0, tsWrapper.ts.dagStats.numNonLocalAllocations);
-      assertEquals(1, tsWrapper.ts.dagStats.numTotalAllocations);
-
-      tsWrapper.resetAppCallback();
-
-      tsWrapper.clock.setTime(10000l);
-      tsWrapper.rejectExecution(task1);
-
-      // Verify that the node is blacklisted
-      assertEquals(1, tsWrapper.ts.dagStats.numRejectedTasks);
-      assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
-      LlapTaskSchedulerService.NodeInfo disabledNodeInfo = tsWrapper.ts.disabledNodesQueue.peek();
-      assertNotNull(disabledNodeInfo);
-      assertEquals(HOST1, disabledNodeInfo.serviceInstance.getHost());
-      assertEquals((10000l), disabledNodeInfo.getDelay(TimeUnit.MILLISECONDS));
-      assertEquals((10000l + 10000l), disabledNodeInfo.expireTimeMillis);
-
-      Object task2 = new Object();
-      Object clientCookie2 = new Object();
-      tsWrapper.allocateTask(task2, hosts1, priority1, clientCookie2);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback).taskAllocated(eq(task2), eq(clientCookie2), any(Container.class));
-      assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
-      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-      assertEquals(1, tsWrapper.ts.dagStats.numNonLocalAllocations);
-      assertEquals(2, tsWrapper.ts.dagStats.numTotalAllocations);
-
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-  @Test(timeout=5000)
-  public void testNodeReEnabled() throws InterruptedException, IOException {
-    // Based on actual timing.
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(1000l);
-    try {
-      Priority priority1 = Priority.newInstance(1);
-      String[] hosts1 = new String[]{HOST1};
-      String[] hosts2 = new String[]{HOST2};
-      String[] hosts3 = new String[]{HOST3};
-
-      Object task1 = new Object();
-      Object clientCookie1 = new Object();
-      Object task2 = new Object();
-      Object clientCookie2 = new Object();
-      Object task3 = new Object();
-      Object clientCookie3 = new Object();
-
-      tsWrapper.controlScheduler(true);
-      tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
-      tsWrapper.allocateTask(task2, hosts2, priority1, clientCookie2);
-      tsWrapper.allocateTask(task3, hosts3, priority1, clientCookie3);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), any(Container.class));
-      assertEquals(3, tsWrapper.ts.dagStats.numLocalAllocations);
-      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-      assertEquals(3, tsWrapper.ts.dagStats.numTotalAllocations);
-
-      tsWrapper.resetAppCallback();
-
-      tsWrapper.rejectExecution(task1);
-      tsWrapper.rejectExecution(task2);
-      tsWrapper.rejectExecution(task3);
-
-      // Verify that the node is blacklisted
-      assertEquals(3, tsWrapper.ts.dagStats.numRejectedTasks);
-      assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
-      assertEquals(3, tsWrapper.ts.disabledNodesQueue.size());
-
-
-      Object task4 = new Object();
-      Object clientCookie4 = new Object();
-      Object task5 = new Object();
-      Object clientCookie5 = new Object();
-      Object task6 = new Object();
-      Object clientCookie6 = new Object();
-      tsWrapper.allocateTask(task4, hosts1, priority1, clientCookie4);
-      tsWrapper.allocateTask(task5, hosts2, priority1, clientCookie5);
-      tsWrapper.allocateTask(task6, hosts3, priority1, clientCookie6);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 6) {
-          break;
-        }
-      }
-
-      ArgumentCaptor<Container> argumentCaptor = ArgumentCaptor.forClass(Container.class);
-      verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), argumentCaptor.capture());
-
-      // which affects the locality matching
-      assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-      assertEquals(6, tsWrapper.ts.dagStats.numTotalAllocations);
-
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-  @Test (timeout = 5000)
-  public void testForceLocalityTest1() throws IOException, InterruptedException {
-    // 2 hosts. 2 per host. 5 requests at the same priority.
-    // First 3 on host1, Next at host2, Last with no host.
-    // Third request on host1 should not be allocated immediately.
-    forceLocalityTest1(true);
-
-  }
-
-  @Test (timeout = 5000)
-  public void testNoForceLocalityCounterTest1() throws IOException, InterruptedException {
-    // 2 hosts. 2 per host. 5 requests at the same priority.
-    // First 3 on host1, Next at host2, Last with no host.
-    // Third should allocate on host2, 4th on host2, 5th will wait.
-
-    forceLocalityTest1(false);
-  }
-
-  private void forceLocalityTest1(boolean forceLocality) throws IOException, InterruptedException {
-    Priority priority1 = Priority.newInstance(1);
-
-    String[] hosts = new String[] {HOST1, HOST2};
-
-    String[] hostsH1 = new String[] {HOST1};
-    String[] hostsH2 = new String[] {HOST2};
-
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, (forceLocality ? -1l : 0l));
-
-    try {
-      Object task1 = "task1";
-      Object clientCookie1 = "cookie1";
-      Object task2 = "task2";
-      Object clientCookie2 = "cookie2";
-      Object task3 = "task3";
-      Object clientCookie3 = "cookie3";
-      Object task4 = "task4";
-      Object clientCookie4 = "cookie4";
-      Object task5 = "task5";
-      Object clientCookie5 = "cookie5";
-
-      tsWrapper.controlScheduler(true);
-      //H1 - should allocate
-      tsWrapper.allocateTask(task1, hostsH1, priority1, clientCookie1);
-      //H1 - should allocate
-      tsWrapper.allocateTask(task2, hostsH1, priority1, clientCookie2);
-      //H1 - no capacity if force, should allocate otherwise
-      tsWrapper.allocateTask(task3, hostsH1, priority1, clientCookie3);
-      //H2 - should allocate
-      tsWrapper.allocateTask(task4, hostsH2, priority1, clientCookie4);
-      //No location - should allocate if force, no capacity otherwise
-      tsWrapper.allocateTask(task5, null, priority1, clientCookie5);
-
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 4) {
-          break;
-        }
-      }
-
-      // Verify no preemption requests - since everything is at the same priority
-      verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
-      ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
-      verify(tsWrapper.mockAppCallback, times(4)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
-      assertEquals(4, argumentCaptor.getAllValues().size());
-      assertEquals(task1, argumentCaptor.getAllValues().get(0));
-      assertEquals(task2, argumentCaptor.getAllValues().get(1));
-      if (forceLocality) {
-        // task3 not allocated
-        assertEquals(task4, argumentCaptor.getAllValues().get(2));
-        assertEquals(task5, argumentCaptor.getAllValues().get(3));
-      } else {
-        assertEquals(task3, argumentCaptor.getAllValues().get(2));
-        assertEquals(task4, argumentCaptor.getAllValues().get(3));
-      }
-
-      //Complete one task on host1.
-      tsWrapper.deallocateTask(task1, true, null);
-
-      reset(tsWrapper.mockAppCallback);
-
-      // Try scheduling again.
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 5) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
-      argumentCaptor = ArgumentCaptor.forClass(Object.class);
-      verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
-      assertEquals(1, argumentCaptor.getAllValues().size());
-      if (forceLocality) {
-        assertEquals(task3, argumentCaptor.getAllValues().get(0));
-      } else {
-        assertEquals(task5, argumentCaptor.getAllValues().get(0));
-      }
-
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-  @Test(timeout = 5000)
-  public void testForcedLocalityUnknownHost() throws IOException, InterruptedException {
-    Priority priority1 = Priority.newInstance(1);
-
-    String[] hostsKnown = new String[]{HOST1};
-    String[] hostsUnknown = new String[]{HOST2};
-
-    TestTaskSchedulerServiceWrapper tsWrapper =
-        new TestTaskSchedulerServiceWrapper(2000, hostsKnown, 1, 1, -1l);
-    try {
-      Object task1 = "task1";
-      Object clientCookie1 = "cookie1";
-
-      Object task2 = "task2";
-      Object clientCookie2 = "cookie2";
-
-      tsWrapper.controlScheduler(true);
-      // Should allocate since H2 is not known.
-      tsWrapper.allocateTask(task1, hostsUnknown, priority1, clientCookie1);
-      tsWrapper.allocateTask(task2, hostsKnown, priority1, clientCookie2);
-
-
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
-          break;
-        }
-      }
-
-      ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
-      verify(tsWrapper.mockAppCallback, times(2))
-          .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
-      assertEquals(2, argumentCaptor.getAllValues().size());
-      assertEquals(task1, argumentCaptor.getAllValues().get(0));
-      assertEquals(task2, argumentCaptor.getAllValues().get(1));
-
-
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-
-  @Test(timeout = 5000)
-  public void testForcedLocalityPreemption() throws IOException, InterruptedException {
-    Priority priority1 = Priority.newInstance(1);
-    Priority priority2 = Priority.newInstance(2);
-    String [] hosts = new String[] {HOST1, HOST2};
-
-    String [] hostsH1 = new String[] {HOST1};
-    String [] hostsH2 = new String[] {HOST2};
-
-    TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, -1l);
-
-    // Fill up host1 with p2 tasks.
-    // Leave host2 empty
-    // Try running p1 task on host1 - should preempt
-
-    try {
-      Object task1 = "task1";
-      Object clientCookie1 = "cookie1";
-      Object task2 = "task2";
-      Object clientCookie2 = "cookie2";
-      Object task3 = "task3";
-      Object clientCookie3 = "cookie3";
-      Object task4 = "task4";
-      Object clientCookie4 = "cookie4";
-
-      tsWrapper.controlScheduler(true);
-      tsWrapper.allocateTask(task1, hostsH1, priority2, clientCookie1);
-      tsWrapper.allocateTask(task2, hostsH1, priority2, clientCookie2);
-      // This request at a lower priority should not affect anything.
-      tsWrapper.allocateTask(task3, hostsH1, priority2, clientCookie3);
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
-          break;
-        }
-      }
-
-      verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
-      ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
-      verify(tsWrapper.mockAppCallback, times(2))
-          .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
-      assertEquals(2, argumentCaptor.getAllValues().size());
-      assertEquals(task1, argumentCaptor.getAllValues().get(0));
-      assertEquals(task2, argumentCaptor.getAllValues().get(1));
-
-      reset(tsWrapper.mockAppCallback);
-      // Allocate t4 at higher priority. t3 should not be allocated,
-      // and a preemption should be attempted on host1, despite host2 having available capacity
-      tsWrapper.allocateTask(task4, hostsH1, priority1, clientCookie4);
-
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
-
-      tsWrapper.deallocateTask(task1, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
-
-      while (true) {
-        tsWrapper.signalSchedulerRun();
-        tsWrapper.awaitSchedulerRun();
-        if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
-          break;
-        }
-      }
-      verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
-          eq(clientCookie4), any(Container.class));
-
-    } finally {
-      tsWrapper.shutdown();
-    }
-  }
-
-  private static class TestTaskSchedulerServiceWrapper {
-    static final Resource resource = Resource.newInstance(1024, 1);
-    Configuration conf;
-    TaskSchedulerContext mockAppCallback = mock(TaskSchedulerContext.class);
-    ControlledClock clock = new ControlledClock(new SystemClock());
-    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1000, 1), 1);
-    LlapTaskSchedulerServiceForTest ts;
-
-    TestTaskSchedulerServiceWrapper() throws IOException, InterruptedException {
-      this(2000l);
-    }
-
-    TestTaskSchedulerServiceWrapper(long disableTimeoutMillis) throws IOException,
-        InterruptedException {
-      this(disableTimeoutMillis, new String[]{HOST1, HOST2, HOST3}, 4,
-          ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.defaultIntVal);
-    }
-
-    TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize) throws
-        IOException, InterruptedException {
-      this(disableTimeoutMillis, hosts, numExecutors, waitQueueSize, 0l);
-    }
-
-    TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize, long localityDelayMs) throws
-        IOException, InterruptedException {
-      conf = new Configuration();
-      conf.setStrings(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, hosts);
-      conf.setInt(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, numExecutors);
-      conf.setInt(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname, waitQueueSize);
-      conf.set(ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS.varname,
-          disableTimeoutMillis + "ms");
-      conf.setBoolean(LlapFixedRegistryImpl.FIXED_REGISTRY_RESOLVE_HOST_NAMES, false);
-      conf.setLong(ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY.varname, localityDelayMs);
-
-      doReturn(appAttemptId).when(mockAppCallback).getApplicationAttemptId();
-      doReturn(11111l).when(mockAppCallback).getCustomClusterIdentifier();
-      UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
-      doReturn(userPayload).when(mockAppCallback).getInitialUserPayload();
-
-      ts = new LlapTaskSchedulerServiceForTest(mockAppCallback, clock);
-
-      controlScheduler(true);
-      ts.initialize();
-      ts.start();
-      // One scheduler pass from the nodes that are added at startup
-      signalSchedulerRun();
-      controlScheduler(false);
-      awaitSchedulerRun();
-    }
-
-    void controlScheduler(boolean val) {
-      ts.forTestsetControlScheduling(val);
-    }
-
-    void signalSchedulerRun() throws InterruptedException {
-      ts.forTestSignalSchedulingRun();
-    }
-
-    void awaitSchedulerRun() throws InterruptedException {
-      ts.forTestAwaitSchedulingRun();
-    }
-    void resetAppCallback() {
-      reset(mockAppCallback);
-    }
-
-    void shutdown() {
-      ts.shutdown();
-    }
-
-    void allocateTask(Object task, String[] hosts, Priority priority, Object clientCookie) {
-      ts.allocateTask(task, resource, hosts, null, priority, null, clientCookie);
-    }
-
-    void deallocateTask(Object task, boolean succeeded, TaskAttemptEndReason endReason) {
-      ts.deallocateTask(task, succeeded, endReason, null);
-    }
-
-    void rejectExecution(Object task) {
-      ts.deallocateTask(task, false, TaskAttemptEndReason.EXECUTOR_BUSY, null);
-    }
-  }
-
-  private static class LlapTaskSchedulerServiceForTest extends LlapTaskSchedulerService {
-
-    private AtomicBoolean controlScheduling = new AtomicBoolean(false);
-    private final Lock testLock = new ReentrantLock();
-    private final Condition schedulingCompleteCondition = testLock.newCondition();
-    private boolean schedulingComplete = false;
-    private final Condition triggerSchedulingCondition = testLock.newCondition();
-    private boolean schedulingTriggered = false;
-    private final AtomicInteger numSchedulerRuns = new AtomicInteger(0);
-
-
-    public LlapTaskSchedulerServiceForTest(
-        TaskSchedulerContext appClient, Clock clock) {
-      super(appClient, clock);
-    }
-
-    @Override
-    protected void schedulePendingTasks() {
-      testLock.lock();
-      try {
-        if (controlScheduling.get()) {
-          while (!schedulingTriggered) {
-            try {
-              triggerSchedulingCondition.await();
-            } catch (InterruptedException e) {
-              throw new RuntimeException(e);
-            }
-          }
-        }
-        numSchedulerRuns.incrementAndGet();
-        super.schedulePendingTasks();
-        schedulingTriggered = false;
-        schedulingComplete = true;
-        schedulingCompleteCondition.signal();
-      } finally {
-        testLock.unlock();
-      }
-    }
-
-    // Enable or disable test scheduling control.
-    void forTestsetControlScheduling(boolean control) {
-      this.controlScheduling.set(control);
-    }
-
-    void forTestSignalSchedulingRun() throws InterruptedException {
-      testLock.lock();
-      try {
-        schedulingTriggered = true;
-        triggerSchedulingCondition.signal();
-      } finally {
-        testLock.unlock();
-      }
-    }
-
-    void forTestAwaitSchedulingRun() throws InterruptedException {
-      testLock.lock();
-      try {
-        while (!schedulingComplete) {
-          schedulingCompleteCondition.await();
-        }
-        schedulingComplete = false;
-      } finally {
-        testLock.unlock();
-      }
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/pom.xml
----------------------------------------------------------------------
diff --git a/llap-tez/pom.xml b/llap-tez/pom.xml
new file mode 100644
index 0000000..ce020da
--- /dev/null
+++ b/llap-tez/pom.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>hive-llap-tez</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Llap Tez</name>
+
+  <properties>
+    <hive.path.to.root>..</hive.path.to.root>
+  </properties>
+
+  <dependencies>
+    <!-- dependencies are always listed in sorted order by groupId, artifactId -->
+    <!-- intra-project -->
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-llap-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <!-- inter-project -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>${commons-lang3.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-api</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-runtime-internals</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-mapreduce</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+      <version>${tez.version}</version>
+      <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- test intra-project -->
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-llap-common</artifactId>
+      <version>${project.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+
+    <!-- test inter-project -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>${mockito-all.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commmons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <sourceDirectory>${basedir}/src/java</sourceDirectory>
+    <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/gen/protobuf/gen-java</source>
+                <source>src/gen/thrift/gen-javabean</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java
new file mode 100644
index 0000000..a314391
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+class ContainerFactory {
+  final ApplicationAttemptId customAppAttemptId;
+  AtomicLong nextId;
+
+  public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) {
+    this.nextId = new AtomicLong(1);
+    ApplicationId appId =
+        ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId());
+    this.customAppAttemptId =
+        ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId());
+  }
+
+  public Container createContainer(Resource capability, Priority priority, String hostname,
+      int port) {
+    ContainerId containerId =
+        ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement());
+    NodeId nodeId = NodeId.newInstance(hostname, port);
+    String nodeHttpAddress = "hostname:0"; // TODO: include UI ports
+
+    Container container =
+        Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null);
+
+    return container;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
new file mode 100644
index 0000000..07703a2
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapContainerLauncher extends ContainerLauncher {
+  private static final Logger LOG = LoggerFactory.getLogger(LlapContainerLauncher.class);
+
+  public LlapContainerLauncher(ContainerLauncherContext containerLauncherContext) {
+    super(containerLauncherContext);
+  }
+
+  @Override
+  public void launchContainer(ContainerLaunchRequest containerLaunchRequest) {
+    LOG.info("No-op launch for container: " + containerLaunchRequest.getContainerId() +
+        " succeeded on host: " + containerLaunchRequest.getNodeId());
+    getContext().containerLaunched(containerLaunchRequest.getContainerId());
+  }
+
+  @Override
+  public void stopContainer(ContainerStopRequest containerStopRequest) {
+    LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + containerStopRequest);
+    getContext().containerStopRequested(containerStopRequest.getContainerId());
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
new file mode 100644
index 0000000..76d095a
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -0,0 +1,757 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
+import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
+import org.apache.hadoop.hive.llap.tez.Converters;
+import org.apache.hadoop.hive.llap.tez.LlapProtocolClientProxy;
+import org.apache.hadoop.hive.llap.tezplugins.helpers.SourceStateTracker;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LlapTaskCommunicator.class);
+
+  private static final boolean isInfoEnabled = LOG.isInfoEnabled();
+  private static final boolean isDebugEnabed = LOG.isDebugEnabled();
+
+  private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
+
+  private final ConcurrentMap<QueryIdentifierProto, ByteBuffer> credentialMap;
+
+  // Tracks containerIds and taskAttemptIds, so can be kept independent of the running DAG.
+  // When DAG specific cleanup happens, it'll be better to link this to a DAG though.
+  private final EntityTracker entityTracker = new EntityTracker();
+  private final SourceStateTracker sourceStateTracker;
+  private final Set<LlapNodeId> nodesForQuery = new HashSet<>();
+
+  private LlapProtocolClientProxy communicator;
+  private long deleteDelayOnDagComplete;
+  private final LlapTaskUmbilicalProtocol umbilical;
+  private final Token<LlapTokenIdentifier> token;
+
+  // These two structures track the list of known nodes, and the list of nodes which are sending in keep-alive heartbeats.
+  // Primarily for debugging purposes a.t.m, since there's some unexplained TASK_TIMEOUTS which are currently being observed.
+  private final ConcurrentMap<LlapNodeId, Long> knownNodeMap = new ConcurrentHashMap<>();
+  private final ConcurrentMap<LlapNodeId, PingingNodeInfo> pingedNodeMap = new ConcurrentHashMap<>();
+
+
+  private volatile int currentDagId;
+  private volatile QueryIdentifierProto currentQueryIdentifierProto;
+
+  public LlapTaskCommunicator(
+      TaskCommunicatorContext taskCommunicatorContext) {
+    super(taskCommunicatorContext);
+    Credentials credentials = taskCommunicatorContext.getCredentials();
+    if (credentials != null) {
+      @SuppressWarnings("unchecked")
+      Token<LlapTokenIdentifier> llapToken =
+          (Token<LlapTokenIdentifier>)credentials.getToken(LlapTokenIdentifier.KIND_NAME);
+      this.token = llapToken;
+    } else {
+      this.token = null;
+    }
+    Preconditions.checkState((token != null) == UserGroupInformation.isSecurityEnabled());
+
+    umbilical = new LlapTaskUmbilicalProtocolImpl(getUmbilical());
+    SubmitWorkRequestProto.Builder baseBuilder = SubmitWorkRequestProto.newBuilder();
+
+    // TODO Avoid reading this from the environment
+    baseBuilder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
+    baseBuilder.setApplicationIdString(
+        taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString());
+    baseBuilder
+        .setAppAttemptNumber(taskCommunicatorContext.getApplicationAttemptId().getAttemptId());
+    baseBuilder.setTokenIdentifier(getTokenIdentifier());
+
+    BASE_SUBMIT_WORK_REQUEST = baseBuilder.build();
+
+    credentialMap = new ConcurrentHashMap<>();
+    sourceStateTracker = new SourceStateTracker(getContext(), this);
+  }
+
+  @Override
+  public void initialize() throws Exception {
+    super.initialize();
+    Configuration conf = getConf();
+    int numThreads = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS);
+    this.communicator = new LlapProtocolClientProxy(numThreads, conf, token);
+    this.deleteDelayOnDagComplete = HiveConf.getTimeVar(
+        conf, ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS);
+    LOG.info("Running LlapTaskCommunicator with "
+        + "fileCleanupDelay=" + deleteDelayOnDagComplete
+        + ", numCommunicatorThreads=" + numThreads);
+    this.communicator.init(conf);
+  }
+
+  @Override
+  public void start() {
+    super.start();
+    this.communicator.start();
+  }
+
+  @Override
+  public void shutdown() {
+    super.shutdown();
+    if (this.communicator != null) {
+      this.communicator.stop();
+    }
+  }
+
+  @Override
+  protected void startRpcServer() {
+    Configuration conf = getConf();
+    try {
+      JobTokenSecretManager jobTokenSecretManager =
+          new JobTokenSecretManager();
+      jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
+
+      int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
+          TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT);
+      server = new RPC.Builder(conf)
+          .setProtocol(LlapTaskUmbilicalProtocol.class)
+          .setBindAddress("0.0.0.0")
+          .setPort(0)
+          .setInstance(umbilical)
+          .setNumHandlers(numHandlers)
+          .setSecretManager(jobTokenSecretManager).build();
+
+      if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
+        server.refreshServiceAcl(conf, new LlapUmbilicalPolicyProvider());
+      }
+
+      server.start();
+      this.address = NetUtils.getConnectAddress(server);
+      LOG.info(
+          "Started LlapUmbilical: " + umbilical.getClass().getName() + " at address: " + address +
+              " with numHandlers=" + numHandlers);
+    } catch (IOException e) {
+      throw new TezUncheckedException(e);
+    }
+  }
+
+  @Override
+  public void registerRunningContainer(ContainerId containerId, String hostname, int port) {
+    super.registerRunningContainer(containerId, hostname, port);
+    entityTracker.registerContainer(containerId, hostname, port);
+
+  }
+
+  @Override
+  public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, String diagnostics) {
+    super.registerContainerEnd(containerId, endReason, diagnostics);
+    if (endReason == ContainerEndReason.INTERNAL_PREEMPTION) {
+      LOG.info("Processing containerEnd for container {} caused by internal preemption", containerId);
+      TezTaskAttemptID taskAttemptId = entityTracker.getTaskAttemptIdForContainer(containerId);
+      if (taskAttemptId != null) {
+        sendTaskTerminated(taskAttemptId, true);
+      }
+    }
+    entityTracker.unregisterContainer(containerId);
+  }
+
+
+  @Override
+  public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
+                                         Map<String, LocalResource> additionalResources,
+                                         Credentials credentials,
+                                         boolean credentialsChanged,
+                                         int priority)  {
+    super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
+        credentialsChanged, priority);
+    int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId();
+    if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIdentifier())) {
+      resetCurrentDag(dagId);
+    }
+
+
+    ContainerInfo containerInfo = getContainerInfo(containerId);
+    String host;
+    int port;
+    if (containerInfo != null) {
+      synchronized (containerInfo) {
+        host = containerInfo.host;
+        port = containerInfo.port;
+      }
+    } else {
+      // TODO Handle this properly
+      throw new RuntimeException("ContainerInfo not found for container: " + containerId +
+          ", while trying to launch task: " + taskSpec.getTaskAttemptID());
+    }
+
+    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+    registerKnownNode(nodeId);
+    entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port);
+    nodesForQuery.add(nodeId);
+
+    sourceStateTracker.registerTaskForStateUpdates(host, port, taskSpec.getInputs());
+    FragmentRuntimeInfo fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo(
+        taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority);
+    SubmitWorkRequestProto requestProto;
+
+    try {
+      requestProto = constructSubmitWorkRequest(containerId, taskSpec, fragmentRuntimeInfo);
+    } catch (IOException e) {
+      throw new RuntimeException("Failed to construct request", e);
+    }
+
+    // Have to register this up front right now. Otherwise, it's possible for the task to start
+    // sending out status/DONE/KILLED/FAILED messages before TAImpl knows how to handle them.
+    getContext()
+        .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
+    communicator.sendSubmitWork(requestProto, host, port,
+        new LlapProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
+          @Override
+          public void setResponse(SubmitWorkResponseProto response) {
+            if (response.hasSubmissionState()) {
+              LlapDaemonProtocolProtos.SubmissionStateProto ss = response.getSubmissionState();
+              if (ss.equals(LlapDaemonProtocolProtos.SubmissionStateProto.REJECTED)) {
+                LOG.info(
+                    "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+                        containerId + ", Service Busy");
+                getContext().taskKilled(taskSpec.getTaskAttemptID(),
+                    TaskAttemptEndReason.EXECUTOR_BUSY, "Service Busy");
+                return;
+              }
+            } else {
+              // TODO: Provide support for reporting errors
+              // This should never happen as server always returns a valid status on success
+              throw new RuntimeException("SubmissionState in response is expected!");
+            }
+            LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
+          }
+
+          @Override
+          public void indicateError(Throwable t) {
+            if (t instanceof ServiceException) {
+              ServiceException se = (ServiceException) t;
+              t = se.getCause();
+            }
+            if (t instanceof RemoteException) {
+              RemoteException re = (RemoteException) t;
+              // All others from the remote service cause the task to FAIL.
+              LOG.info(
+                  "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+                      containerId, t);
+              getContext()
+                  .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
+                      t.toString());
+            } else {
+              // Exception from the RPC layer - communication failure, consider as KILLED / service down.
+              if (t instanceof IOException) {
+                LOG.info(
+                    "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+                        containerId + ", Communication Error");
+                getContext().taskKilled(taskSpec.getTaskAttemptID(),
+                    TaskAttemptEndReason.COMMUNICATION_ERROR, "Communication Error");
+              } else {
+                // Anything else is a FAIL.
+                LOG.info(
+                    "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+                        containerId, t);
+                getContext()
+                    .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
+                        t.getMessage());
+              }
+            }
+          }
+        });
+  }
+
+  @Override
+  public void unregisterRunningTaskAttempt(final TezTaskAttemptID taskAttemptId,
+                                           TaskAttemptEndReason endReason,
+                                           String diagnostics) {
+    super.unregisterRunningTaskAttempt(taskAttemptId, endReason, diagnostics);
+
+    if (endReason == TaskAttemptEndReason.INTERNAL_PREEMPTION) {
+      LOG.info("Processing taskEnd for task {} caused by internal preemption", taskAttemptId);
+      sendTaskTerminated(taskAttemptId, false);
+    }
+    entityTracker.unregisterTaskAttempt(taskAttemptId);
+    // This will also be invoked for tasks which have been KILLED / rejected by the daemon.
+    // Informing the daemon becomes necessary once the LlapScheduler supports preemption
+    // and/or starts attempting to kill tasks which may be running on a node.
+  }
+
+  private void sendTaskTerminated(final TezTaskAttemptID taskAttemptId,
+                                  boolean invokedByContainerEnd) {
+    LOG.info(
+        "DBG: Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}",
+        taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd");
+    LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId);
+    // NodeId can be null if the task gets unregistered due to failure / being killed by the daemon itself
+    if (nodeId != null) {
+      TerminateFragmentRequestProto request =
+          TerminateFragmentRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifierProto)
+              .setFragmentIdentifierString(taskAttemptId.toString()).build();
+      communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(),
+          new LlapProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() {
+            @Override
+            public void setResponse(TerminateFragmentResponseProto response) {
+            }
+
+            @Override
+            public void indicateError(Throwable t) {
+              LOG.warn("Failed to send terminate fragment request for {}",
+                  taskAttemptId.toString());
+            }
+          });
+    } else {
+      LOG.info(
+          "Not sending terminate request for fragment {} since it's node is not known. Already unregistered",
+          taskAttemptId.toString());
+    }
+  }
+
+
+
+
+  @Override
+  public void dagComplete(final int dagIdentifier) {
+    QueryCompleteRequestProto request = QueryCompleteRequestProto.newBuilder()
+        .setQueryIdentifier(constructQueryIdentifierProto(dagIdentifier))
+        .setDeleteDelay(deleteDelayOnDagComplete).build();
+    for (final LlapNodeId llapNodeId : nodesForQuery) {
+      LOG.info("Sending dagComplete message for {}, to {}", dagIdentifier, llapNodeId);
+      communicator.sendQueryComplete(request, llapNodeId.getHostname(), llapNodeId.getPort(),
+          new LlapProtocolClientProxy.ExecuteRequestCallback<LlapDaemonProtocolProtos.QueryCompleteResponseProto>() {
+            @Override
+            public void setResponse(LlapDaemonProtocolProtos.QueryCompleteResponseProto response) {
+            }
+
+            @Override
+            public void indicateError(Throwable t) {
+              LOG.warn("Failed to indicate dag complete dagId={} to node {}", dagIdentifier, llapNodeId);
+            }
+          });
+    }
+
+    nodesForQuery.clear();
+    // TODO Ideally move some of the other cleanup code from resetCurrentDag over here
+  }
+
+  @Override
+  public void onVertexStateUpdated(VertexStateUpdate vertexStateUpdate) {
+    // Delegate updates over to the source state tracker.
+    sourceStateTracker
+        .sourceStateUpdated(vertexStateUpdate.getVertexName(), vertexStateUpdate.getVertexState());
+  }
+
+  public void sendStateUpdate(final String host, final int port,
+                              final SourceStateUpdatedRequestProto request) {
+    communicator.sendSourceStateUpdate(request, host, port,
+        new LlapProtocolClientProxy.ExecuteRequestCallback<SourceStateUpdatedResponseProto>() {
+          @Override
+          public void setResponse(SourceStateUpdatedResponseProto response) {
+          }
+
+          @Override
+          public void indicateError(Throwable t) {
+            // TODO HIVE-10280.
+            // Ideally, this should be retried for a while, after which the node should be marked as failed.
+            // Considering tasks are supposed to run fast. Failing the task immediately may be a good option.
+            LOG.error(
+                "Failed to send state update to node: " + host + ":" + port + ", StateUpdate=" +
+                    request, t);
+          }
+        });
+  }
+
+
+  private static class PingingNodeInfo {
+    final AtomicLong logTimestamp;
+    final AtomicInteger pingCount;
+
+    PingingNodeInfo(long currentTs) {
+      logTimestamp = new AtomicLong(currentTs);
+      pingCount = new AtomicInteger(1);
+    }
+  }
+
+  public void registerKnownNode(LlapNodeId nodeId) {
+    Long old = knownNodeMap.putIfAbsent(nodeId,
+        TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS));
+    if (old == null) {
+      if (isInfoEnabled) {
+        LOG.info("Added new known node: {}", nodeId);
+      }
+    }
+  }
+
+  public void registerPingingNode(LlapNodeId nodeId) {
+    long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
+    PingingNodeInfo ni = new PingingNodeInfo(currentTs);
+    PingingNodeInfo old = pingedNodeMap.put(nodeId, ni);
+    if (old == null) {
+      if (isInfoEnabled) {
+        LOG.info("Added new pinging node: [{}]", nodeId);
+      }
+    } else {
+      old.pingCount.incrementAndGet();
+    }
+    // The node should always be known by this point. Log occasionally if it is not known.
+    if (!knownNodeMap.containsKey(nodeId)) {
+      if (old == null) {
+        // First time this is seen. Log it.
+        LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, ni.pingCount.get());
+      } else {
+        // Pinged before. Log only occasionally.
+        if (currentTs > old.logTimestamp.get() + 5000l) { // 5 seconds elapsed. Log again.
+          LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, old.pingCount.get());
+          old.logTimestamp.set(currentTs);
+        }
+      }
+
+    }
+  }
+
+
+  private final AtomicLong nodeNotFoundLogTime = new AtomicLong(0);
+
+  void nodePinged(String hostname, int port) {
+    LlapNodeId nodeId = LlapNodeId.getInstance(hostname, port);
+    registerPingingNode(nodeId);
+    BiMap<ContainerId, TezTaskAttemptID> biMap =
+        entityTracker.getContainerAttemptMapForNode(nodeId);
+    if (biMap != null) {
+      synchronized (biMap) {
+        for (Map.Entry<ContainerId, TezTaskAttemptID> entry : biMap.entrySet()) {
+          getContext().taskAlive(entry.getValue());
+          getContext().containerAlive(entry.getKey());
+        }
+      }
+    } else {
+      long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
+      if (currentTs > nodeNotFoundLogTime.get() + 5000l) {
+        LOG.warn("Received ping from node without any registered tasks or containers: " + hostname +
+            ":" + port +
+            ". Could be caused by pre-emption by the AM," +
+            " or a mismatched hostname. Enable debug logging for mismatched host names");
+        nodeNotFoundLogTime.set(currentTs);
+      }
+    }
+  }
+
+  private void resetCurrentDag(int newDagId) {
+    // Working on the assumption that a single DAG runs at a time per AM.
+    currentQueryIdentifierProto = constructQueryIdentifierProto(newDagId);
+    sourceStateTracker.resetState(newDagId);
+    nodesForQuery.clear();
+    LOG.info("CurrentDagId set to: " + newDagId + ", name=" + getContext().getCurrentDagName());
+    // TODO Is it possible for heartbeats to come in from lost tasks - those should be told to die, which
+    // is likely already happening.
+  }
+
+  private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId,
+                                                            TaskSpec taskSpec,
+                                                            FragmentRuntimeInfo fragmentRuntimeInfo) throws
+      IOException {
+    SubmitWorkRequestProto.Builder builder =
+        SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
+    builder.setContainerIdString(containerId.toString());
+    builder.setAmHost(getAddress().getHostName());
+    builder.setAmPort(getAddress().getPort());
+    Credentials taskCredentials = new Credentials();
+    // Credentials can change across DAGs. Ideally construct only once per DAG.
+    taskCredentials.addAll(getContext().getCredentials());
+
+    Preconditions.checkState(currentQueryIdentifierProto.getDagIdentifier() ==
+        taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId());
+    ByteBuffer credentialsBinary = credentialMap.get(currentQueryIdentifierProto);
+    if (credentialsBinary == null) {
+      credentialsBinary = serializeCredentials(getContext().getCredentials());
+      credentialMap.putIfAbsent(currentQueryIdentifierProto, credentialsBinary.duplicate());
+    } else {
+      credentialsBinary = credentialsBinary.duplicate();
+    }
+    builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
+    builder.setFragmentSpec(Converters.convertTaskSpecToProto(taskSpec));
+    builder.setFragmentRuntimeInfo(fragmentRuntimeInfo);
+    return builder.build();
+  }
+
+  private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
+    Credentials containerCredentials = new Credentials();
+    containerCredentials.addAll(credentials);
+    DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
+    containerCredentials.writeTokenStorageToStream(containerTokens_dob);
+    return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
+  }
+
+
+
+  protected class LlapTaskUmbilicalProtocolImpl implements LlapTaskUmbilicalProtocol {
+
+    private final TezTaskUmbilicalProtocol tezUmbilical;
+
+    public LlapTaskUmbilicalProtocolImpl(TezTaskUmbilicalProtocol tezUmbilical) {
+      this.tezUmbilical = tezUmbilical;
+    }
+
+    @Override
+    public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
+      return tezUmbilical.canCommit(taskid);
+    }
+
+    @Override
+    public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
+        TezException {
+      return tezUmbilical.heartbeat(request);
+    }
+
+    @Override
+    public void nodeHeartbeat(Text hostname, int port) throws IOException {
+      nodePinged(hostname.toString(), port);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Received heartbeat from [" + hostname + ":" + port +"]");
+      }
+    }
+
+    @Override
+    public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException {
+      // TODO Unregister the task for state updates, which could in turn unregister the node.
+      getContext().taskKilled(taskAttemptId,
+          TaskAttemptEndReason.EXTERNAL_PREEMPTION, "Attempt preempted");
+      entityTracker.unregisterTaskAttempt(taskAttemptId);
+    }
+
+    @Override
+    public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+      return versionID;
+    }
+
+    @Override
+    public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+                                                  int clientMethodsHash) throws IOException {
+      return ProtocolSignature.getProtocolSignature(this, protocol,
+          clientVersion, clientMethodsHash);
+    }
+  }
+
+  /**
+   * Track the association between known containers and taskAttempts, along with the nodes they are assigned to.
+   */
+  @VisibleForTesting
+  static final class EntityTracker {
+    @VisibleForTesting
+    final ConcurrentMap<TezTaskAttemptID, LlapNodeId> attemptToNodeMap = new ConcurrentHashMap<>();
+    @VisibleForTesting
+    final ConcurrentMap<ContainerId, LlapNodeId> containerToNodeMap = new ConcurrentHashMap<>();
+    @VisibleForTesting
+    final ConcurrentMap<LlapNodeId, BiMap<ContainerId, TezTaskAttemptID>> nodeMap = new ConcurrentHashMap<>();
+
+    void registerTaskAttempt(ContainerId containerId, TezTaskAttemptID taskAttemptId, String host, int port) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Registering " + containerId + ", " + taskAttemptId + " for node: " + host + ":" + port);
+      }
+      LlapNodeId llapNodeId = LlapNodeId.getInstance(host, port);
+      attemptToNodeMap.putIfAbsent(taskAttemptId, llapNodeId);
+
+      registerContainer(containerId, host, port);
+
+      // nodeMap registration.
+      BiMap<ContainerId, TezTaskAttemptID> tmpMap = HashBiMap.create();
+      BiMap<ContainerId, TezTaskAttemptID> old = nodeMap.putIfAbsent(llapNodeId, tmpMap);
+      BiMap<ContainerId, TezTaskAttemptID> usedInstance;
+      usedInstance = old == null ? tmpMap : old;
+      synchronized(usedInstance) {
+        usedInstance.put(containerId, taskAttemptId);
+      }
+      // Make sure to put the instance back again, in case it was removed as part of a
+      // containerEnd/taskEnd invocation.
+      nodeMap.putIfAbsent(llapNodeId, usedInstance);
+    }
+
+    void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
+      LlapNodeId llapNodeId = attemptToNodeMap.remove(attemptId);
+      if (llapNodeId == null) {
+        // Possible since either container / task can be unregistered.
+        return;
+      }
+
+      BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
+      ContainerId matched = null;
+      if (bMap != null) {
+        synchronized(bMap) {
+          matched = bMap.inverse().remove(attemptId);
+        }
+        if (bMap.isEmpty()) {
+          nodeMap.remove(llapNodeId);
+        }
+      }
+
+      // Remove the container mapping
+      if (matched != null) {
+        containerToNodeMap.remove(matched);
+      }
+
+    }
+
+    void registerContainer(ContainerId containerId, String hostname, int port) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Registering " + containerId + " for node: " + hostname + ":" + port);
+      }
+      containerToNodeMap.putIfAbsent(containerId, LlapNodeId.getInstance(hostname, port));
+      // nodeMap registration is not required, since there's no taskId association.
+    }
+
+    LlapNodeId getNodeIdForContainer(ContainerId containerId) {
+      return containerToNodeMap.get(containerId);
+    }
+
+    LlapNodeId getNodeIdForTaskAttempt(TezTaskAttemptID taskAttemptId) {
+      return attemptToNodeMap.get(taskAttemptId);
+    }
+
+    ContainerId getContainerIdForAttempt(TezTaskAttemptID taskAttemptId) {
+      LlapNodeId llapNodeId = getNodeIdForTaskAttempt(taskAttemptId);
+      if (llapNodeId != null) {
+        BiMap<TezTaskAttemptID, ContainerId> bMap = nodeMap.get(llapNodeId).inverse();
+        if (bMap != null) {
+          synchronized (bMap) {
+            return bMap.get(taskAttemptId);
+          }
+        } else {
+          return null;
+        }
+      } else {
+        return null;
+      }
+    }
+
+    TezTaskAttemptID getTaskAttemptIdForContainer(ContainerId containerId) {
+      LlapNodeId llapNodeId = getNodeIdForContainer(containerId);
+      if (llapNodeId != null) {
+        BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
+        if (bMap != null) {
+          synchronized (bMap) {
+            return bMap.get(containerId);
+          }
+        } else {
+          return null;
+        }
+      } else {
+        return null;
+      }
+    }
+
+    void unregisterContainer(ContainerId containerId) {
+      LlapNodeId llapNodeId = containerToNodeMap.remove(containerId);
+      if (llapNodeId == null) {
+        // Possible since either container / task can be unregistered.
+        return;
+      }
+
+      BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
+      TezTaskAttemptID matched = null;
+      if (bMap != null) {
+        synchronized(bMap) {
+          matched = bMap.remove(containerId);
+        }
+        if (bMap.isEmpty()) {
+          nodeMap.remove(llapNodeId);
+        }
+      }
+
+      // Remove the container mapping
+      if (matched != null) {
+        attemptToNodeMap.remove(matched);
+      }
+    }
+
+    /**
+     * Return a {@link BiMap} containing container->taskAttemptId mapping for the host specified.
+     * </p>
+     * <p/>
+     * This method return the internal structure used by the EntityTracker. Users must synchronize
+     * on the structure to ensure correct usage.
+     *
+     * @param llapNodeId
+     * @return
+     */
+    BiMap<ContainerId, TezTaskAttemptID> getContainerAttemptMapForNode(LlapNodeId llapNodeId) {
+      BiMap<ContainerId, TezTaskAttemptID> biMap = nodeMap.get(llapNodeId);
+      return biMap;
+    }
+
+  }
+
+  private QueryIdentifierProto constructQueryIdentifierProto(int dagIdentifier) {
+    return QueryIdentifierProto.newBuilder()
+        .setAppIdentifier(getContext().getCurrentAppIdentifier()).setDagIdentifier(dagIdentifier)
+        .build();
+  }
+}


[07/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
----------------------------------------------------------------------
diff --git a/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java b/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
deleted file mode 100644
index 4ab7b32..0000000
--- a/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
+++ /dev/null
@@ -1,14553 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: LlapDaemonProtocol.proto
-
-package org.apache.hadoop.hive.llap.daemon.rpc;
-
-public final class LlapDaemonProtocolProtos {
-  private LlapDaemonProtocolProtos() {}
-  public static void registerAllExtensions(
-      com.google.protobuf.ExtensionRegistry registry) {
-  }
-  /**
-   * Protobuf enum {@code SourceStateProto}
-   */
-  public enum SourceStateProto
-      implements com.google.protobuf.ProtocolMessageEnum {
-    /**
-     * <code>S_SUCCEEDED = 1;</code>
-     */
-    S_SUCCEEDED(0, 1),
-    /**
-     * <code>S_RUNNING = 2;</code>
-     */
-    S_RUNNING(1, 2),
-    ;
-
-    /**
-     * <code>S_SUCCEEDED = 1;</code>
-     */
-    public static final int S_SUCCEEDED_VALUE = 1;
-    /**
-     * <code>S_RUNNING = 2;</code>
-     */
-    public static final int S_RUNNING_VALUE = 2;
-
-
-    public final int getNumber() { return value; }
-
-    public static SourceStateProto valueOf(int value) {
-      switch (value) {
-        case 1: return S_SUCCEEDED;
-        case 2: return S_RUNNING;
-        default: return null;
-      }
-    }
-
-    public static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
-        internalGetValueMap() {
-      return internalValueMap;
-    }
-    private static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
-        internalValueMap =
-          new com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>() {
-            public SourceStateProto findValueByNumber(int number) {
-              return SourceStateProto.valueOf(number);
-            }
-          };
-
-    public final com.google.protobuf.Descriptors.EnumValueDescriptor
-        getValueDescriptor() {
-      return getDescriptor().getValues().get(index);
-    }
-    public final com.google.protobuf.Descriptors.EnumDescriptor
-        getDescriptorForType() {
-      return getDescriptor();
-    }
-    public static final com.google.protobuf.Descriptors.EnumDescriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(0);
-    }
-
-    private static final SourceStateProto[] VALUES = values();
-
-    public static SourceStateProto valueOf(
-        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
-      if (desc.getType() != getDescriptor()) {
-        throw new java.lang.IllegalArgumentException(
-          "EnumValueDescriptor is not for this type.");
-      }
-      return VALUES[desc.getIndex()];
-    }
-
-    private final int index;
-    private final int value;
-
-    private SourceStateProto(int index, int value) {
-      this.index = index;
-      this.value = value;
-    }
-
-    // @@protoc_insertion_point(enum_scope:SourceStateProto)
-  }
-
-  /**
-   * Protobuf enum {@code SubmissionStateProto}
-   */
-  public enum SubmissionStateProto
-      implements com.google.protobuf.ProtocolMessageEnum {
-    /**
-     * <code>ACCEPTED = 1;</code>
-     */
-    ACCEPTED(0, 1),
-    /**
-     * <code>REJECTED = 2;</code>
-     */
-    REJECTED(1, 2),
-    /**
-     * <code>EVICTED_OTHER = 3;</code>
-     */
-    EVICTED_OTHER(2, 3),
-    ;
-
-    /**
-     * <code>ACCEPTED = 1;</code>
-     */
-    public static final int ACCEPTED_VALUE = 1;
-    /**
-     * <code>REJECTED = 2;</code>
-     */
-    public static final int REJECTED_VALUE = 2;
-    /**
-     * <code>EVICTED_OTHER = 3;</code>
-     */
-    public static final int EVICTED_OTHER_VALUE = 3;
-
-
-    public final int getNumber() { return value; }
-
-    public static SubmissionStateProto valueOf(int value) {
-      switch (value) {
-        case 1: return ACCEPTED;
-        case 2: return REJECTED;
-        case 3: return EVICTED_OTHER;
-        default: return null;
-      }
-    }
-
-    public static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
-        internalGetValueMap() {
-      return internalValueMap;
-    }
-    private static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
-        internalValueMap =
-          new com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>() {
-            public SubmissionStateProto findValueByNumber(int number) {
-              return SubmissionStateProto.valueOf(number);
-            }
-          };
-
-    public final com.google.protobuf.Descriptors.EnumValueDescriptor
-        getValueDescriptor() {
-      return getDescriptor().getValues().get(index);
-    }
-    public final com.google.protobuf.Descriptors.EnumDescriptor
-        getDescriptorForType() {
-      return getDescriptor();
-    }
-    public static final com.google.protobuf.Descriptors.EnumDescriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(1);
-    }
-
-    private static final SubmissionStateProto[] VALUES = values();
-
-    public static SubmissionStateProto valueOf(
-        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
-      if (desc.getType() != getDescriptor()) {
-        throw new java.lang.IllegalArgumentException(
-          "EnumValueDescriptor is not for this type.");
-      }
-      return VALUES[desc.getIndex()];
-    }
-
-    private final int index;
-    private final int value;
-
-    private SubmissionStateProto(int index, int value) {
-      this.index = index;
-      this.value = value;
-    }
-
-    // @@protoc_insertion_point(enum_scope:SubmissionStateProto)
-  }
-
-  public interface UserPayloadProtoOrBuilder
-      extends com.google.protobuf.MessageOrBuilder {
-
-    // optional bytes user_payload = 1;
-    /**
-     * <code>optional bytes user_payload = 1;</code>
-     */
-    boolean hasUserPayload();
-    /**
-     * <code>optional bytes user_payload = 1;</code>
-     */
-    com.google.protobuf.ByteString getUserPayload();
-
-    // optional int32 version = 2;
-    /**
-     * <code>optional int32 version = 2;</code>
-     */
-    boolean hasVersion();
-    /**
-     * <code>optional int32 version = 2;</code>
-     */
-    int getVersion();
-  }
-  /**
-   * Protobuf type {@code UserPayloadProto}
-   */
-  public static final class UserPayloadProto extends
-      com.google.protobuf.GeneratedMessage
-      implements UserPayloadProtoOrBuilder {
-    // Use UserPayloadProto.newBuilder() to construct.
-    private UserPayloadProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-      super(builder);
-      this.unknownFields = builder.getUnknownFields();
-    }
-    private UserPayloadProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-    private static final UserPayloadProto defaultInstance;
-    public static UserPayloadProto getDefaultInstance() {
-      return defaultInstance;
-    }
-
-    public UserPayloadProto getDefaultInstanceForType() {
-      return defaultInstance;
-    }
-
-    private final com.google.protobuf.UnknownFieldSet unknownFields;
-    @java.lang.Override
-    public final com.google.protobuf.UnknownFieldSet
-        getUnknownFields() {
-      return this.unknownFields;
-    }
-    private UserPayloadProto(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      initFields();
-      int mutable_bitField0_ = 0;
-      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-          com.google.protobuf.UnknownFieldSet.newBuilder();
-      try {
-        boolean done = false;
-        while (!done) {
-          int tag = input.readTag();
-          switch (tag) {
-            case 0:
-              done = true;
-              break;
-            default: {
-              if (!parseUnknownField(input, unknownFields,
-                                     extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
-            case 10: {
-              bitField0_ |= 0x00000001;
-              userPayload_ = input.readBytes();
-              break;
-            }
-            case 16: {
-              bitField0_ |= 0x00000002;
-              version_ = input.readInt32();
-              break;
-            }
-          }
-        }
-      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-        throw e.setUnfinishedMessage(this);
-      } catch (java.io.IOException e) {
-        throw new com.google.protobuf.InvalidProtocolBufferException(
-            e.getMessage()).setUnfinishedMessage(this);
-      } finally {
-        this.unknownFields = unknownFields.build();
-        makeExtensionsImmutable();
-      }
-    }
-    public static final com.google.protobuf.Descriptors.Descriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
-    }
-
-    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-        internalGetFieldAccessorTable() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
-          .ensureFieldAccessorsInitialized(
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
-    }
-
-    public static com.google.protobuf.Parser<UserPayloadProto> PARSER =
-        new com.google.protobuf.AbstractParser<UserPayloadProto>() {
-      public UserPayloadProto parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new UserPayloadProto(input, extensionRegistry);
-      }
-    };
-
-    @java.lang.Override
-    public com.google.protobuf.Parser<UserPayloadProto> getParserForType() {
-      return PARSER;
-    }
-
-    private int bitField0_;
-    // optional bytes user_payload = 1;
-    public static final int USER_PAYLOAD_FIELD_NUMBER = 1;
-    private com.google.protobuf.ByteString userPayload_;
-    /**
-     * <code>optional bytes user_payload = 1;</code>
-     */
-    public boolean hasUserPayload() {
-      return ((bitField0_ & 0x00000001) == 0x00000001);
-    }
-    /**
-     * <code>optional bytes user_payload = 1;</code>
-     */
-    public com.google.protobuf.ByteString getUserPayload() {
-      return userPayload_;
-    }
-
-    // optional int32 version = 2;
-    public static final int VERSION_FIELD_NUMBER = 2;
-    private int version_;
-    /**
-     * <code>optional int32 version = 2;</code>
-     */
-    public boolean hasVersion() {
-      return ((bitField0_ & 0x00000002) == 0x00000002);
-    }
-    /**
-     * <code>optional int32 version = 2;</code>
-     */
-    public int getVersion() {
-      return version_;
-    }
-
-    private void initFields() {
-      userPayload_ = com.google.protobuf.ByteString.EMPTY;
-      version_ = 0;
-    }
-    private byte memoizedIsInitialized = -1;
-    public final boolean isInitialized() {
-      byte isInitialized = memoizedIsInitialized;
-      if (isInitialized != -1) return isInitialized == 1;
-
-      memoizedIsInitialized = 1;
-      return true;
-    }
-
-    public void writeTo(com.google.protobuf.CodedOutputStream output)
-                        throws java.io.IOException {
-      getSerializedSize();
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        output.writeBytes(1, userPayload_);
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        output.writeInt32(2, version_);
-      }
-      getUnknownFields().writeTo(output);
-    }
-
-    private int memoizedSerializedSize = -1;
-    public int getSerializedSize() {
-      int size = memoizedSerializedSize;
-      if (size != -1) return size;
-
-      size = 0;
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeBytesSize(1, userPayload_);
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeInt32Size(2, version_);
-      }
-      size += getUnknownFields().getSerializedSize();
-      memoizedSerializedSize = size;
-      return size;
-    }
-
-    private static final long serialVersionUID = 0L;
-    @java.lang.Override
-    protected java.lang.Object writeReplace()
-        throws java.io.ObjectStreamException {
-      return super.writeReplace();
-    }
-
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)) {
-        return super.equals(obj);
-      }
-      org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) obj;
-
-      boolean result = true;
-      result = result && (hasUserPayload() == other.hasUserPayload());
-      if (hasUserPayload()) {
-        result = result && getUserPayload()
-            .equals(other.getUserPayload());
-      }
-      result = result && (hasVersion() == other.hasVersion());
-      if (hasVersion()) {
-        result = result && (getVersion()
-            == other.getVersion());
-      }
-      result = result &&
-          getUnknownFields().equals(other.getUnknownFields());
-      return result;
-    }
-
-    private int memoizedHashCode = 0;
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptorForType().hashCode();
-      if (hasUserPayload()) {
-        hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
-        hash = (53 * hash) + getUserPayload().hashCode();
-      }
-      if (hasVersion()) {
-        hash = (37 * hash) + VERSION_FIELD_NUMBER;
-        hash = (53 * hash) + getVersion();
-      }
-      hash = (29 * hash) + getUnknownFields().hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
-        com.google.protobuf.ByteString data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
-        com.google.protobuf.ByteString data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(byte[] data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
-        byte[] data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
-        com.google.protobuf.CodedInputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-
-    public static Builder newBuilder() { return Builder.create(); }
-    public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto prototype) {
-      return newBuilder().mergeFrom(prototype);
-    }
-    public Builder toBuilder() { return newBuilder(this); }
-
-    @java.lang.Override
-    protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-      Builder builder = new Builder(parent);
-      return builder;
-    }
-    /**
-     * Protobuf type {@code UserPayloadProto}
-     */
-    public static final class Builder extends
-        com.google.protobuf.GeneratedMessage.Builder<Builder>
-       implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder {
-      public static final com.google.protobuf.Descriptors.Descriptor
-          getDescriptor() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
-      }
-
-      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-          internalGetFieldAccessorTable() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
-            .ensureFieldAccessorsInitialized(
-                org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
-      }
-
-      // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder()
-      private Builder() {
-        maybeForceBuilderInitialization();
-      }
-
-      private Builder(
-          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-        super(parent);
-        maybeForceBuilderInitialization();
-      }
-      private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-        }
-      }
-      private static Builder create() {
-        return new Builder();
-      }
-
-      public Builder clear() {
-        super.clear();
-        userPayload_ = com.google.protobuf.ByteString.EMPTY;
-        bitField0_ = (bitField0_ & ~0x00000001);
-        version_ = 0;
-        bitField0_ = (bitField0_ & ~0x00000002);
-        return this;
-      }
-
-      public Builder clone() {
-        return create().mergeFrom(buildPartial());
-      }
-
-      public com.google.protobuf.Descriptors.Descriptor
-          getDescriptorForType() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getDefaultInstanceForType() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto build() {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = buildPartial();
-        if (!result.isInitialized()) {
-          throw newUninitializedMessageException(result);
-        }
-        return result;
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto buildPartial() {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto(this);
-        int from_bitField0_ = bitField0_;
-        int to_bitField0_ = 0;
-        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
-          to_bitField0_ |= 0x00000001;
-        }
-        result.userPayload_ = userPayload_;
-        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
-          to_bitField0_ |= 0x00000002;
-        }
-        result.version_ = version_;
-        result.bitField0_ = to_bitField0_;
-        onBuilt();
-        return result;
-      }
-
-      public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) {
-          return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)other);
-        } else {
-          super.mergeFrom(other);
-          return this;
-        }
-      }
-
-      public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other) {
-        if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) return this;
-        if (other.hasUserPayload()) {
-          setUserPayload(other.getUserPayload());
-        }
-        if (other.hasVersion()) {
-          setVersion(other.getVersion());
-        }
-        this.mergeUnknownFields(other.getUnknownFields());
-        return this;
-      }
-
-      public final boolean isInitialized() {
-        return true;
-      }
-
-      public Builder mergeFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parsedMessage = null;
-        try {
-          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
-        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) e.getUnfinishedMessage();
-          throw e;
-        } finally {
-          if (parsedMessage != null) {
-            mergeFrom(parsedMessage);
-          }
-        }
-        return this;
-      }
-      private int bitField0_;
-
-      // optional bytes user_payload = 1;
-      private com.google.protobuf.ByteString userPayload_ = com.google.protobuf.ByteString.EMPTY;
-      /**
-       * <code>optional bytes user_payload = 1;</code>
-       */
-      public boolean hasUserPayload() {
-        return ((bitField0_ & 0x00000001) == 0x00000001);
-      }
-      /**
-       * <code>optional bytes user_payload = 1;</code>
-       */
-      public com.google.protobuf.ByteString getUserPayload() {
-        return userPayload_;
-      }
-      /**
-       * <code>optional bytes user_payload = 1;</code>
-       */
-      public Builder setUserPayload(com.google.protobuf.ByteString value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000001;
-        userPayload_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional bytes user_payload = 1;</code>
-       */
-      public Builder clearUserPayload() {
-        bitField0_ = (bitField0_ & ~0x00000001);
-        userPayload_ = getDefaultInstance().getUserPayload();
-        onChanged();
-        return this;
-      }
-
-      // optional int32 version = 2;
-      private int version_ ;
-      /**
-       * <code>optional int32 version = 2;</code>
-       */
-      public boolean hasVersion() {
-        return ((bitField0_ & 0x00000002) == 0x00000002);
-      }
-      /**
-       * <code>optional int32 version = 2;</code>
-       */
-      public int getVersion() {
-        return version_;
-      }
-      /**
-       * <code>optional int32 version = 2;</code>
-       */
-      public Builder setVersion(int value) {
-        bitField0_ |= 0x00000002;
-        version_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional int32 version = 2;</code>
-       */
-      public Builder clearVersion() {
-        bitField0_ = (bitField0_ & ~0x00000002);
-        version_ = 0;
-        onChanged();
-        return this;
-      }
-
-      // @@protoc_insertion_point(builder_scope:UserPayloadProto)
-    }
-
-    static {
-      defaultInstance = new UserPayloadProto(true);
-      defaultInstance.initFields();
-    }
-
-    // @@protoc_insertion_point(class_scope:UserPayloadProto)
-  }
-
-  public interface EntityDescriptorProtoOrBuilder
-      extends com.google.protobuf.MessageOrBuilder {
-
-    // optional string class_name = 1;
-    /**
-     * <code>optional string class_name = 1;</code>
-     */
-    boolean hasClassName();
-    /**
-     * <code>optional string class_name = 1;</code>
-     */
-    java.lang.String getClassName();
-    /**
-     * <code>optional string class_name = 1;</code>
-     */
-    com.google.protobuf.ByteString
-        getClassNameBytes();
-
-    // optional .UserPayloadProto user_payload = 2;
-    /**
-     * <code>optional .UserPayloadProto user_payload = 2;</code>
-     */
-    boolean hasUserPayload();
-    /**
-     * <code>optional .UserPayloadProto user_payload = 2;</code>
-     */
-    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload();
-    /**
-     * <code>optional .UserPayloadProto user_payload = 2;</code>
-     */
-    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder();
-
-    // optional bytes history_text = 3;
-    /**
-     * <code>optional bytes history_text = 3;</code>
-     */
-    boolean hasHistoryText();
-    /**
-     * <code>optional bytes history_text = 3;</code>
-     */
-    com.google.protobuf.ByteString getHistoryText();
-  }
-  /**
-   * Protobuf type {@code EntityDescriptorProto}
-   */
-  public static final class EntityDescriptorProto extends
-      com.google.protobuf.GeneratedMessage
-      implements EntityDescriptorProtoOrBuilder {
-    // Use EntityDescriptorProto.newBuilder() to construct.
-    private EntityDescriptorProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-      super(builder);
-      this.unknownFields = builder.getUnknownFields();
-    }
-    private EntityDescriptorProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-    private static final EntityDescriptorProto defaultInstance;
-    public static EntityDescriptorProto getDefaultInstance() {
-      return defaultInstance;
-    }
-
-    public EntityDescriptorProto getDefaultInstanceForType() {
-      return defaultInstance;
-    }
-
-    private final com.google.protobuf.UnknownFieldSet unknownFields;
-    @java.lang.Override
-    public final com.google.protobuf.UnknownFieldSet
-        getUnknownFields() {
-      return this.unknownFields;
-    }
-    private EntityDescriptorProto(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      initFields();
-      int mutable_bitField0_ = 0;
-      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-          com.google.protobuf.UnknownFieldSet.newBuilder();
-      try {
-        boolean done = false;
-        while (!done) {
-          int tag = input.readTag();
-          switch (tag) {
-            case 0:
-              done = true;
-              break;
-            default: {
-              if (!parseUnknownField(input, unknownFields,
-                                     extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
-            case 10: {
-              bitField0_ |= 0x00000001;
-              className_ = input.readBytes();
-              break;
-            }
-            case 18: {
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder subBuilder = null;
-              if (((bitField0_ & 0x00000002) == 0x00000002)) {
-                subBuilder = userPayload_.toBuilder();
-              }
-              userPayload_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.PARSER, extensionRegistry);
-              if (subBuilder != null) {
-                subBuilder.mergeFrom(userPayload_);
-                userPayload_ = subBuilder.buildPartial();
-              }
-              bitField0_ |= 0x00000002;
-              break;
-            }
-            case 26: {
-              bitField0_ |= 0x00000004;
-              historyText_ = input.readBytes();
-              break;
-            }
-          }
-        }
-      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-        throw e.setUnfinishedMessage(this);
-      } catch (java.io.IOException e) {
-        throw new com.google.protobuf.InvalidProtocolBufferException(
-            e.getMessage()).setUnfinishedMessage(this);
-      } finally {
-        this.unknownFields = unknownFields.build();
-        makeExtensionsImmutable();
-      }
-    }
-    public static final com.google.protobuf.Descriptors.Descriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
-    }
-
-    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-        internalGetFieldAccessorTable() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
-          .ensureFieldAccessorsInitialized(
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
-    }
-
-    public static com.google.protobuf.Parser<EntityDescriptorProto> PARSER =
-        new com.google.protobuf.AbstractParser<EntityDescriptorProto>() {
-      public EntityDescriptorProto parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new EntityDescriptorProto(input, extensionRegistry);
-      }
-    };
-
-    @java.lang.Override
-    public com.google.protobuf.Parser<EntityDescriptorProto> getParserForType() {
-      return PARSER;
-    }
-
-    private int bitField0_;
-    // optional string class_name = 1;
-    public static final int CLASS_NAME_FIELD_NUMBER = 1;
-    private java.lang.Object className_;
-    /**
-     * <code>optional string class_name = 1;</code>
-     */
-    public boolean hasClassName() {
-      return ((bitField0_ & 0x00000001) == 0x00000001);
-    }
-    /**
-     * <code>optional string class_name = 1;</code>
-     */
-    public java.lang.String getClassName() {
-      java.lang.Object ref = className_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
-      } else {
-        com.google.protobuf.ByteString bs = 
-            (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        if (bs.isValidUtf8()) {
-          className_ = s;
-        }
-        return s;
-      }
-    }
-    /**
-     * <code>optional string class_name = 1;</code>
-     */
-    public com.google.protobuf.ByteString
-        getClassNameBytes() {
-      java.lang.Object ref = className_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
-            com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
-        className_ = b;
-        return b;
-      } else {
-        return (com.google.protobuf.ByteString) ref;
-      }
-    }
-
-    // optional .UserPayloadProto user_payload = 2;
-    public static final int USER_PAYLOAD_FIELD_NUMBER = 2;
-    private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_;
-    /**
-     * <code>optional .UserPayloadProto user_payload = 2;</code>
-     */
-    public boolean hasUserPayload() {
-      return ((bitField0_ & 0x00000002) == 0x00000002);
-    }
-    /**
-     * <code>optional .UserPayloadProto user_payload = 2;</code>
-     */
-    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
-      return userPayload_;
-    }
-    /**
-     * <code>optional .UserPayloadProto user_payload = 2;</code>
-     */
-    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
-      return userPayload_;
-    }
-
-    // optional bytes history_text = 3;
-    public static final int HISTORY_TEXT_FIELD_NUMBER = 3;
-    private com.google.protobuf.ByteString historyText_;
-    /**
-     * <code>optional bytes history_text = 3;</code>
-     */
-    public boolean hasHistoryText() {
-      return ((bitField0_ & 0x00000004) == 0x00000004);
-    }
-    /**
-     * <code>optional bytes history_text = 3;</code>
-     */
-    public com.google.protobuf.ByteString getHistoryText() {
-      return historyText_;
-    }
-
-    private void initFields() {
-      className_ = "";
-      userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
-      historyText_ = com.google.protobuf.ByteString.EMPTY;
-    }
-    private byte memoizedIsInitialized = -1;
-    public final boolean isInitialized() {
-      byte isInitialized = memoizedIsInitialized;
-      if (isInitialized != -1) return isInitialized == 1;
-
-      memoizedIsInitialized = 1;
-      return true;
-    }
-
-    public void writeTo(com.google.protobuf.CodedOutputStream output)
-                        throws java.io.IOException {
-      getSerializedSize();
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        output.writeBytes(1, getClassNameBytes());
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        output.writeMessage(2, userPayload_);
-      }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
-        output.writeBytes(3, historyText_);
-      }
-      getUnknownFields().writeTo(output);
-    }
-
-    private int memoizedSerializedSize = -1;
-    public int getSerializedSize() {
-      int size = memoizedSerializedSize;
-      if (size != -1) return size;
-
-      size = 0;
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeBytesSize(1, getClassNameBytes());
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeMessageSize(2, userPayload_);
-      }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeBytesSize(3, historyText_);
-      }
-      size += getUnknownFields().getSerializedSize();
-      memoizedSerializedSize = size;
-      return size;
-    }
-
-    private static final long serialVersionUID = 0L;
-    @java.lang.Override
-    protected java.lang.Object writeReplace()
-        throws java.io.ObjectStreamException {
-      return super.writeReplace();
-    }
-
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)) {
-        return super.equals(obj);
-      }
-      org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) obj;
-
-      boolean result = true;
-      result = result && (hasClassName() == other.hasClassName());
-      if (hasClassName()) {
-        result = result && getClassName()
-            .equals(other.getClassName());
-      }
-      result = result && (hasUserPayload() == other.hasUserPayload());
-      if (hasUserPayload()) {
-        result = result && getUserPayload()
-            .equals(other.getUserPayload());
-      }
-      result = result && (hasHistoryText() == other.hasHistoryText());
-      if (hasHistoryText()) {
-        result = result && getHistoryText()
-            .equals(other.getHistoryText());
-      }
-      result = result &&
-          getUnknownFields().equals(other.getUnknownFields());
-      return result;
-    }
-
-    private int memoizedHashCode = 0;
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptorForType().hashCode();
-      if (hasClassName()) {
-        hash = (37 * hash) + CLASS_NAME_FIELD_NUMBER;
-        hash = (53 * hash) + getClassName().hashCode();
-      }
-      if (hasUserPayload()) {
-        hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
-        hash = (53 * hash) + getUserPayload().hashCode();
-      }
-      if (hasHistoryText()) {
-        hash = (37 * hash) + HISTORY_TEXT_FIELD_NUMBER;
-        hash = (53 * hash) + getHistoryText().hashCode();
-      }
-      hash = (29 * hash) + getUnknownFields().hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
-        com.google.protobuf.ByteString data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
-        com.google.protobuf.ByteString data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(byte[] data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
-        byte[] data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
-        com.google.protobuf.CodedInputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-
-    public static Builder newBuilder() { return Builder.create(); }
-    public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto prototype) {
-      return newBuilder().mergeFrom(prototype);
-    }
-    public Builder toBuilder() { return newBuilder(this); }
-
-    @java.lang.Override
-    protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-      Builder builder = new Builder(parent);
-      return builder;
-    }
-    /**
-     * Protobuf type {@code EntityDescriptorProto}
-     */
-    public static final class Builder extends
-        com.google.protobuf.GeneratedMessage.Builder<Builder>
-       implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder {
-      public static final com.google.protobuf.Descriptors.Descriptor
-          getDescriptor() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
-      }
-
-      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-          internalGetFieldAccessorTable() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
-            .ensureFieldAccessorsInitialized(
-                org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
-      }
-
-      // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder()
-      private Builder() {
-        maybeForceBuilderInitialization();
-      }
-
-      private Builder(
-          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-        super(parent);
-        maybeForceBuilderInitialization();
-      }
-      private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-          getUserPayloadFieldBuilder();
-        }
-      }
-      private static Builder create() {
-        return new Builder();
-      }
-
-      public Builder clear() {
-        super.clear();
-        className_ = "";
-        bitField0_ = (bitField0_ & ~0x00000001);
-        if (userPayloadBuilder_ == null) {
-          userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
-        } else {
-          userPayloadBuilder_.clear();
-        }
-        bitField0_ = (bitField0_ & ~0x00000002);
-        historyText_ = com.google.protobuf.ByteString.EMPTY;
-        bitField0_ = (bitField0_ & ~0x00000004);
-        return this;
-      }
-
-      public Builder clone() {
-        return create().mergeFrom(buildPartial());
-      }
-
-      public com.google.protobuf.Descriptors.Descriptor
-          getDescriptorForType() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getDefaultInstanceForType() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto build() {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = buildPartial();
-        if (!result.isInitialized()) {
-          throw newUninitializedMessageException(result);
-        }
-        return result;
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto buildPartial() {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto(this);
-        int from_bitField0_ = bitField0_;
-        int to_bitField0_ = 0;
-        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
-          to_bitField0_ |= 0x00000001;
-        }
-        result.className_ = className_;
-        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
-          to_bitField0_ |= 0x00000002;
-        }
-        if (userPayloadBuilder_ == null) {
-          result.userPayload_ = userPayload_;
-        } else {
-          result.userPayload_ = userPayloadBuilder_.build();
-        }
-        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
-          to_bitField0_ |= 0x00000004;
-        }
-        result.historyText_ = historyText_;
-        result.bitField0_ = to_bitField0_;
-        onBuilt();
-        return result;
-      }
-
-      public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) {
-          return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)other);
-        } else {
-          super.mergeFrom(other);
-          return this;
-        }
-      }
-
-      public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other) {
-        if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) return this;
-        if (other.hasClassName()) {
-          bitField0_ |= 0x00000001;
-          className_ = other.className_;
-          onChanged();
-        }
-        if (other.hasUserPayload()) {
-          mergeUserPayload(other.getUserPayload());
-        }
-        if (other.hasHistoryText()) {
-          setHistoryText(other.getHistoryText());
-        }
-        this.mergeUnknownFields(other.getUnknownFields());
-        return this;
-      }
-
-      public final boolean isInitialized() {
-        return true;
-      }
-
-      public Builder mergeFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parsedMessage = null;
-        try {
-          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
-        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) e.getUnfinishedMessage();
-          throw e;
-        } finally {
-          if (parsedMessage != null) {
-            mergeFrom(parsedMessage);
-          }
-        }
-        return this;
-      }
-      private int bitField0_;
-
-      // optional string class_name = 1;
-      private java.lang.Object className_ = "";
-      /**
-       * <code>optional string class_name = 1;</code>
-       */
-      public boolean hasClassName() {
-        return ((bitField0_ & 0x00000001) == 0x00000001);
-      }
-      /**
-       * <code>optional string class_name = 1;</code>
-       */
-      public java.lang.String getClassName() {
-        java.lang.Object ref = className_;
-        if (!(ref instanceof java.lang.String)) {
-          java.lang.String s = ((com.google.protobuf.ByteString) ref)
-              .toStringUtf8();
-          className_ = s;
-          return s;
-        } else {
-          return (java.lang.String) ref;
-        }
-      }
-      /**
-       * <code>optional string class_name = 1;</code>
-       */
-      public com.google.protobuf.ByteString
-          getClassNameBytes() {
-        java.lang.Object ref = className_;
-        if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
-              com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
-          className_ = b;
-          return b;
-        } else {
-          return (com.google.protobuf.ByteString) ref;
-        }
-      }
-      /**
-       * <code>optional string class_name = 1;</code>
-       */
-      public Builder setClassName(
-          java.lang.String value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000001;
-        className_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional string class_name = 1;</code>
-       */
-      public Builder clearClassName() {
-        bitField0_ = (bitField0_ & ~0x00000001);
-        className_ = getDefaultInstance().getClassName();
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional string class_name = 1;</code>
-       */
-      public Builder setClassNameBytes(
-          com.google.protobuf.ByteString value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000001;
-        className_ = value;
-        onChanged();
-        return this;
-      }
-
-      // optional .UserPayloadProto user_payload = 2;
-      private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
-      private com.google.protobuf.SingleFieldBuilder<
-          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder> userPayloadBuilder_;
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public boolean hasUserPayload() {
-        return ((bitField0_ & 0x00000002) == 0x00000002);
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
-        if (userPayloadBuilder_ == null) {
-          return userPayload_;
-        } else {
-          return userPayloadBuilder_.getMessage();
-        }
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public Builder setUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
-        if (userPayloadBuilder_ == null) {
-          if (value == null) {
-            throw new NullPointerException();
-          }
-          userPayload_ = value;
-          onChanged();
-        } else {
-          userPayloadBuilder_.setMessage(value);
-        }
-        bitField0_ |= 0x00000002;
-        return this;
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public Builder setUserPayload(
-          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder builderForValue) {
-        if (userPayloadBuilder_ == null) {
-          userPayload_ = builderForValue.build();
-          onChanged();
-        } else {
-          userPayloadBuilder_.setMessage(builderForValue.build());
-        }
-        bitField0_ |= 0x00000002;
-        return this;
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public Builder mergeUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
-        if (userPayloadBuilder_ == null) {
-          if (((bitField0_ & 0x00000002) == 0x00000002) &&
-              userPayload_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) {
-            userPayload_ =
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder(userPayload_).mergeFrom(value).buildPartial();
-          } else {
-            userPayload_ = value;
-          }
-          onChanged();
-        } else {
-          userPayloadBuilder_.mergeFrom(value);
-        }
-        bitField0_ |= 0x00000002;
-        return this;
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public Builder clearUserPayload() {
-        if (userPayloadBuilder_ == null) {
-          userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
-          onChanged();
-        } else {
-          userPayloadBuilder_.clear();
-        }
-        bitField0_ = (bitField0_ & ~0x00000002);
-        return this;
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder getUserPayloadBuilder() {
-        bitField0_ |= 0x00000002;
-        onChanged();
-        return getUserPayloadFieldBuilder().getBuilder();
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
-        if (userPayloadBuilder_ != null) {
-          return userPayloadBuilder_.getMessageOrBuilder();
-        } else {
-          return userPayload_;
-        }
-      }
-      /**
-       * <code>optional .UserPayloadProto user_payload = 2;</code>
-       */
-      private com.google.protobuf.SingleFieldBuilder<
-          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder> 
-          getUserPayloadFieldBuilder() {
-        if (userPayloadBuilder_ == null) {
-          userPayloadBuilder_ = new com.google.protobuf.SingleFieldBuilder<
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder>(
-                  userPayload_,
-                  getParentForChildren(),
-                  isClean());
-          userPayload_ = null;
-        }
-        return userPayloadBuilder_;
-      }
-
-      // optional bytes history_text = 3;
-      private com.google.protobuf.ByteString historyText_ = com.google.protobuf.ByteString.EMPTY;
-      /**
-       * <code>optional bytes history_text = 3;</code>
-       */
-      public boolean hasHistoryText() {
-        return ((bitField0_ & 0x00000004) == 0x00000004);
-      }
-      /**
-       * <code>optional bytes history_text = 3;</code>
-       */
-      public com.google.protobuf.ByteString getHistoryText() {
-        return historyText_;
-      }
-      /**
-       * <code>optional bytes history_text = 3;</code>
-       */
-      public Builder setHistoryText(com.google.protobuf.ByteString value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000004;
-        historyText_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional bytes history_text = 3;</code>
-       */
-      public Builder clearHistoryText() {
-        bitField0_ = (bitField0_ & ~0x00000004);
-        historyText_ = getDefaultInstance().getHistoryText();
-        onChanged();
-        return this;
-      }
-
-      // @@protoc_insertion_point(builder_scope:EntityDescriptorProto)
-    }
-
-    static {
-      defaultInstance = new EntityDescriptorProto(true);
-      defaultInstance.initFields();
-    }
-
-    // @@protoc_insertion_point(class_scope:EntityDescriptorProto)
-  }
-
-  public interface IOSpecProtoOrBuilder
-      extends com.google.protobuf.MessageOrBuilder {
-
-    // optional string connected_vertex_name = 1;
-    /**
-     * <code>optional string connected_vertex_name = 1;</code>
-     */
-    boolean hasConnectedVertexName();
-    /**
-     * <code>optional string connected_vertex_name = 1;</code>
-     */
-    java.lang.String getConnectedVertexName();
-    /**
-     * <code>optional string connected_vertex_name = 1;</code>
-     */
-    com.google.protobuf.ByteString
-        getConnectedVertexNameBytes();
-
-    // optional .EntityDescriptorProto io_descriptor = 2;
-    /**
-     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-     */
-    boolean hasIoDescriptor();
-    /**
-     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-     */
-    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor();
-    /**
-     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-     */
-    org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder();
-
-    // optional int32 physical_edge_count = 3;
-    /**
-     * <code>optional int32 physical_edge_count = 3;</code>
-     */
-    boolean hasPhysicalEdgeCount();
-    /**
-     * <code>optional int32 physical_edge_count = 3;</code>
-     */
-    int getPhysicalEdgeCount();
-  }
-  /**
-   * Protobuf type {@code IOSpecProto}
-   */
-  public static final class IOSpecProto extends
-      com.google.protobuf.GeneratedMessage
-      implements IOSpecProtoOrBuilder {
-    // Use IOSpecProto.newBuilder() to construct.
-    private IOSpecProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
-      super(builder);
-      this.unknownFields = builder.getUnknownFields();
-    }
-    private IOSpecProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-    private static final IOSpecProto defaultInstance;
-    public static IOSpecProto getDefaultInstance() {
-      return defaultInstance;
-    }
-
-    public IOSpecProto getDefaultInstanceForType() {
-      return defaultInstance;
-    }
-
-    private final com.google.protobuf.UnknownFieldSet unknownFields;
-    @java.lang.Override
-    public final com.google.protobuf.UnknownFieldSet
-        getUnknownFields() {
-      return this.unknownFields;
-    }
-    private IOSpecProto(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      initFields();
-      int mutable_bitField0_ = 0;
-      com.google.protobuf.UnknownFieldSet.Builder unknownFields =
-          com.google.protobuf.UnknownFieldSet.newBuilder();
-      try {
-        boolean done = false;
-        while (!done) {
-          int tag = input.readTag();
-          switch (tag) {
-            case 0:
-              done = true;
-              break;
-            default: {
-              if (!parseUnknownField(input, unknownFields,
-                                     extensionRegistry, tag)) {
-                done = true;
-              }
-              break;
-            }
-            case 10: {
-              bitField0_ |= 0x00000001;
-              connectedVertexName_ = input.readBytes();
-              break;
-            }
-            case 18: {
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder subBuilder = null;
-              if (((bitField0_ & 0x00000002) == 0x00000002)) {
-                subBuilder = ioDescriptor_.toBuilder();
-              }
-              ioDescriptor_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.PARSER, extensionRegistry);
-              if (subBuilder != null) {
-                subBuilder.mergeFrom(ioDescriptor_);
-                ioDescriptor_ = subBuilder.buildPartial();
-              }
-              bitField0_ |= 0x00000002;
-              break;
-            }
-            case 24: {
-              bitField0_ |= 0x00000004;
-              physicalEdgeCount_ = input.readInt32();
-              break;
-            }
-          }
-        }
-      } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-        throw e.setUnfinishedMessage(this);
-      } catch (java.io.IOException e) {
-        throw new com.google.protobuf.InvalidProtocolBufferException(
-            e.getMessage()).setUnfinishedMessage(this);
-      } finally {
-        this.unknownFields = unknownFields.build();
-        makeExtensionsImmutable();
-      }
-    }
-    public static final com.google.protobuf.Descriptors.Descriptor
-        getDescriptor() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
-    }
-
-    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-        internalGetFieldAccessorTable() {
-      return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
-          .ensureFieldAccessorsInitialized(
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
-    }
-
-    public static com.google.protobuf.Parser<IOSpecProto> PARSER =
-        new com.google.protobuf.AbstractParser<IOSpecProto>() {
-      public IOSpecProto parsePartialFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws com.google.protobuf.InvalidProtocolBufferException {
-        return new IOSpecProto(input, extensionRegistry);
-      }
-    };
-
-    @java.lang.Override
-    public com.google.protobuf.Parser<IOSpecProto> getParserForType() {
-      return PARSER;
-    }
-
-    private int bitField0_;
-    // optional string connected_vertex_name = 1;
-    public static final int CONNECTED_VERTEX_NAME_FIELD_NUMBER = 1;
-    private java.lang.Object connectedVertexName_;
-    /**
-     * <code>optional string connected_vertex_name = 1;</code>
-     */
-    public boolean hasConnectedVertexName() {
-      return ((bitField0_ & 0x00000001) == 0x00000001);
-    }
-    /**
-     * <code>optional string connected_vertex_name = 1;</code>
-     */
-    public java.lang.String getConnectedVertexName() {
-      java.lang.Object ref = connectedVertexName_;
-      if (ref instanceof java.lang.String) {
-        return (java.lang.String) ref;
-      } else {
-        com.google.protobuf.ByteString bs = 
-            (com.google.protobuf.ByteString) ref;
-        java.lang.String s = bs.toStringUtf8();
-        if (bs.isValidUtf8()) {
-          connectedVertexName_ = s;
-        }
-        return s;
-      }
-    }
-    /**
-     * <code>optional string connected_vertex_name = 1;</code>
-     */
-    public com.google.protobuf.ByteString
-        getConnectedVertexNameBytes() {
-      java.lang.Object ref = connectedVertexName_;
-      if (ref instanceof java.lang.String) {
-        com.google.protobuf.ByteString b = 
-            com.google.protobuf.ByteString.copyFromUtf8(
-                (java.lang.String) ref);
-        connectedVertexName_ = b;
-        return b;
-      } else {
-        return (com.google.protobuf.ByteString) ref;
-      }
-    }
-
-    // optional .EntityDescriptorProto io_descriptor = 2;
-    public static final int IO_DESCRIPTOR_FIELD_NUMBER = 2;
-    private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_;
-    /**
-     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-     */
-    public boolean hasIoDescriptor() {
-      return ((bitField0_ & 0x00000002) == 0x00000002);
-    }
-    /**
-     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-     */
-    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
-      return ioDescriptor_;
-    }
-    /**
-     * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-     */
-    public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
-      return ioDescriptor_;
-    }
-
-    // optional int32 physical_edge_count = 3;
-    public static final int PHYSICAL_EDGE_COUNT_FIELD_NUMBER = 3;
-    private int physicalEdgeCount_;
-    /**
-     * <code>optional int32 physical_edge_count = 3;</code>
-     */
-    public boolean hasPhysicalEdgeCount() {
-      return ((bitField0_ & 0x00000004) == 0x00000004);
-    }
-    /**
-     * <code>optional int32 physical_edge_count = 3;</code>
-     */
-    public int getPhysicalEdgeCount() {
-      return physicalEdgeCount_;
-    }
-
-    private void initFields() {
-      connectedVertexName_ = "";
-      ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
-      physicalEdgeCount_ = 0;
-    }
-    private byte memoizedIsInitialized = -1;
-    public final boolean isInitialized() {
-      byte isInitialized = memoizedIsInitialized;
-      if (isInitialized != -1) return isInitialized == 1;
-
-      memoizedIsInitialized = 1;
-      return true;
-    }
-
-    public void writeTo(com.google.protobuf.CodedOutputStream output)
-                        throws java.io.IOException {
-      getSerializedSize();
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        output.writeBytes(1, getConnectedVertexNameBytes());
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        output.writeMessage(2, ioDescriptor_);
-      }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
-        output.writeInt32(3, physicalEdgeCount_);
-      }
-      getUnknownFields().writeTo(output);
-    }
-
-    private int memoizedSerializedSize = -1;
-    public int getSerializedSize() {
-      int size = memoizedSerializedSize;
-      if (size != -1) return size;
-
-      size = 0;
-      if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeBytesSize(1, getConnectedVertexNameBytes());
-      }
-      if (((bitField0_ & 0x00000002) == 0x00000002)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeMessageSize(2, ioDescriptor_);
-      }
-      if (((bitField0_ & 0x00000004) == 0x00000004)) {
-        size += com.google.protobuf.CodedOutputStream
-          .computeInt32Size(3, physicalEdgeCount_);
-      }
-      size += getUnknownFields().getSerializedSize();
-      memoizedSerializedSize = size;
-      return size;
-    }
-
-    private static final long serialVersionUID = 0L;
-    @java.lang.Override
-    protected java.lang.Object writeReplace()
-        throws java.io.ObjectStreamException {
-      return super.writeReplace();
-    }
-
-    @java.lang.Override
-    public boolean equals(final java.lang.Object obj) {
-      if (obj == this) {
-       return true;
-      }
-      if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)) {
-        return super.equals(obj);
-      }
-      org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) obj;
-
-      boolean result = true;
-      result = result && (hasConnectedVertexName() == other.hasConnectedVertexName());
-      if (hasConnectedVertexName()) {
-        result = result && getConnectedVertexName()
-            .equals(other.getConnectedVertexName());
-      }
-      result = result && (hasIoDescriptor() == other.hasIoDescriptor());
-      if (hasIoDescriptor()) {
-        result = result && getIoDescriptor()
-            .equals(other.getIoDescriptor());
-      }
-      result = result && (hasPhysicalEdgeCount() == other.hasPhysicalEdgeCount());
-      if (hasPhysicalEdgeCount()) {
-        result = result && (getPhysicalEdgeCount()
-            == other.getPhysicalEdgeCount());
-      }
-      result = result &&
-          getUnknownFields().equals(other.getUnknownFields());
-      return result;
-    }
-
-    private int memoizedHashCode = 0;
-    @java.lang.Override
-    public int hashCode() {
-      if (memoizedHashCode != 0) {
-        return memoizedHashCode;
-      }
-      int hash = 41;
-      hash = (19 * hash) + getDescriptorForType().hashCode();
-      if (hasConnectedVertexName()) {
-        hash = (37 * hash) + CONNECTED_VERTEX_NAME_FIELD_NUMBER;
-        hash = (53 * hash) + getConnectedVertexName().hashCode();
-      }
-      if (hasIoDescriptor()) {
-        hash = (37 * hash) + IO_DESCRIPTOR_FIELD_NUMBER;
-        hash = (53 * hash) + getIoDescriptor().hashCode();
-      }
-      if (hasPhysicalEdgeCount()) {
-        hash = (37 * hash) + PHYSICAL_EDGE_COUNT_FIELD_NUMBER;
-        hash = (53 * hash) + getPhysicalEdgeCount();
-      }
-      hash = (29 * hash) + getUnknownFields().hashCode();
-      memoizedHashCode = hash;
-      return hash;
-    }
-
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
-        com.google.protobuf.ByteString data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
-        com.google.protobuf.ByteString data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(byte[] data)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
-        byte[] data,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws com.google.protobuf.InvalidProtocolBufferException {
-      return PARSER.parseFrom(data, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(java.io.InputStream input)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(
-        java.io.InputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseDelimitedFrom(input, extensionRegistry);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
-        com.google.protobuf.CodedInputStream input)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input);
-    }
-    public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
-        com.google.protobuf.CodedInputStream input,
-        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-        throws java.io.IOException {
-      return PARSER.parseFrom(input, extensionRegistry);
-    }
-
-    public static Builder newBuilder() { return Builder.create(); }
-    public Builder newBuilderForType() { return newBuilder(); }
-    public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto prototype) {
-      return newBuilder().mergeFrom(prototype);
-    }
-    public Builder toBuilder() { return newBuilder(this); }
-
-    @java.lang.Override
-    protected Builder newBuilderForType(
-        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-      Builder builder = new Builder(parent);
-      return builder;
-    }
-    /**
-     * Protobuf type {@code IOSpecProto}
-     */
-    public static final class Builder extends
-        com.google.protobuf.GeneratedMessage.Builder<Builder>
-       implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProtoOrBuilder {
-      public static final com.google.protobuf.Descriptors.Descriptor
-          getDescriptor() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
-      }
-
-      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
-          internalGetFieldAccessorTable() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
-            .ensureFieldAccessorsInitialized(
-                org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
-      }
-
-      // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.newBuilder()
-      private Builder() {
-        maybeForceBuilderInitialization();
-      }
-
-      private Builder(
-          com.google.protobuf.GeneratedMessage.BuilderParent parent) {
-        super(parent);
-        maybeForceBuilderInitialization();
-      }
-      private void maybeForceBuilderInitialization() {
-        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
-          getIoDescriptorFieldBuilder();
-        }
-      }
-      private static Builder create() {
-        return new Builder();
-      }
-
-      public Builder clear() {
-        super.clear();
-        connectedVertexName_ = "";
-        bitField0_ = (bitField0_ & ~0x00000001);
-        if (ioDescriptorBuilder_ == null) {
-          ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
-        } else {
-          ioDescriptorBuilder_.clear();
-        }
-        bitField0_ = (bitField0_ & ~0x00000002);
-        physicalEdgeCount_ = 0;
-        bitField0_ = (bitField0_ & ~0x00000004);
-        return this;
-      }
-
-      public Builder clone() {
-        return create().mergeFrom(buildPartial());
-      }
-
-      public com.google.protobuf.Descriptors.Descriptor
-          getDescriptorForType() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto getDefaultInstanceForType() {
-        return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance();
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto build() {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = buildPartial();
-        if (!result.isInitialized()) {
-          throw newUninitializedMessageException(result);
-        }
-        return result;
-      }
-
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto buildPartial() {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto(this);
-        int from_bitField0_ = bitField0_;
-        int to_bitField0_ = 0;
-        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
-          to_bitField0_ |= 0x00000001;
-        }
-        result.connectedVertexName_ = connectedVertexName_;
-        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
-          to_bitField0_ |= 0x00000002;
-        }
-        if (ioDescriptorBuilder_ == null) {
-          result.ioDescriptor_ = ioDescriptor_;
-        } else {
-          result.ioDescriptor_ = ioDescriptorBuilder_.build();
-        }
-        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
-          to_bitField0_ |= 0x00000004;
-        }
-        result.physicalEdgeCount_ = physicalEdgeCount_;
-        result.bitField0_ = to_bitField0_;
-        onBuilt();
-        return result;
-      }
-
-      public Builder mergeFrom(com.google.protobuf.Message other) {
-        if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) {
-          return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)other);
-        } else {
-          super.mergeFrom(other);
-          return this;
-        }
-      }
-
-      public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other) {
-        if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance()) return this;
-        if (other.hasConnectedVertexName()) {
-          bitField0_ |= 0x00000001;
-          connectedVertexName_ = other.connectedVertexName_;
-          onChanged();
-        }
-        if (other.hasIoDescriptor()) {
-          mergeIoDescriptor(other.getIoDescriptor());
-        }
-        if (other.hasPhysicalEdgeCount()) {
-          setPhysicalEdgeCount(other.getPhysicalEdgeCount());
-        }
-        this.mergeUnknownFields(other.getUnknownFields());
-        return this;
-      }
-
-      public final boolean isInitialized() {
-        return true;
-      }
-
-      public Builder mergeFrom(
-          com.google.protobuf.CodedInputStream input,
-          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
-          throws java.io.IOException {
-        org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parsedMessage = null;
-        try {
-          parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
-        } catch (com.google.protobuf.InvalidProtocolBufferException e) {
-          parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) e.getUnfinishedMessage();
-          throw e;
-        } finally {
-          if (parsedMessage != null) {
-            mergeFrom(parsedMessage);
-          }
-        }
-        return this;
-      }
-      private int bitField0_;
-
-      // optional string connected_vertex_name = 1;
-      private java.lang.Object connectedVertexName_ = "";
-      /**
-       * <code>optional string connected_vertex_name = 1;</code>
-       */
-      public boolean hasConnectedVertexName() {
-        return ((bitField0_ & 0x00000001) == 0x00000001);
-      }
-      /**
-       * <code>optional string connected_vertex_name = 1;</code>
-       */
-      public java.lang.String getConnectedVertexName() {
-        java.lang.Object ref = connectedVertexName_;
-        if (!(ref instanceof java.lang.String)) {
-          java.lang.String s = ((com.google.protobuf.ByteString) ref)
-              .toStringUtf8();
-          connectedVertexName_ = s;
-          return s;
-        } else {
-          return (java.lang.String) ref;
-        }
-      }
-      /**
-       * <code>optional string connected_vertex_name = 1;</code>
-       */
-      public com.google.protobuf.ByteString
-          getConnectedVertexNameBytes() {
-        java.lang.Object ref = connectedVertexName_;
-        if (ref instanceof String) {
-          com.google.protobuf.ByteString b = 
-              com.google.protobuf.ByteString.copyFromUtf8(
-                  (java.lang.String) ref);
-          connectedVertexName_ = b;
-          return b;
-        } else {
-          return (com.google.protobuf.ByteString) ref;
-        }
-      }
-      /**
-       * <code>optional string connected_vertex_name = 1;</code>
-       */
-      public Builder setConnectedVertexName(
-          java.lang.String value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000001;
-        connectedVertexName_ = value;
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional string connected_vertex_name = 1;</code>
-       */
-      public Builder clearConnectedVertexName() {
-        bitField0_ = (bitField0_ & ~0x00000001);
-        connectedVertexName_ = getDefaultInstance().getConnectedVertexName();
-        onChanged();
-        return this;
-      }
-      /**
-       * <code>optional string connected_vertex_name = 1;</code>
-       */
-      public Builder setConnectedVertexNameBytes(
-          com.google.protobuf.ByteString value) {
-        if (value == null) {
-    throw new NullPointerException();
-  }
-  bitField0_ |= 0x00000001;
-        connectedVertexName_ = value;
-        onChanged();
-        return this;
-      }
-
-      // optional .EntityDescriptorProto io_descriptor = 2;
-      private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
-      private com.google.protobuf.SingleFieldBuilder<
-          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder> ioDescriptorBuilder_;
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public boolean hasIoDescriptor() {
-        return ((bitField0_ & 0x00000002) == 0x00000002);
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
-        if (ioDescriptorBuilder_ == null) {
-          return ioDescriptor_;
-        } else {
-          return ioDescriptorBuilder_.getMessage();
-        }
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public Builder setIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
-        if (ioDescriptorBuilder_ == null) {
-          if (value == null) {
-            throw new NullPointerException();
-          }
-          ioDescriptor_ = value;
-          onChanged();
-        } else {
-          ioDescriptorBuilder_.setMessage(value);
-        }
-        bitField0_ |= 0x00000002;
-        return this;
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public Builder setIoDescriptor(
-          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder builderForValue) {
-        if (ioDescriptorBuilder_ == null) {
-          ioDescriptor_ = builderForValue.build();
-          onChanged();
-        } else {
-          ioDescriptorBuilder_.setMessage(builderForValue.build());
-        }
-        bitField0_ |= 0x00000002;
-        return this;
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public Builder mergeIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
-        if (ioDescriptorBuilder_ == null) {
-          if (((bitField0_ & 0x00000002) == 0x00000002) &&
-              ioDescriptor_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) {
-            ioDescriptor_ =
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder(ioDescriptor_).mergeFrom(value).buildPartial();
-          } else {
-            ioDescriptor_ = value;
-          }
-          onChanged();
-        } else {
-          ioDescriptorBuilder_.mergeFrom(value);
-        }
-        bitField0_ |= 0x00000002;
-        return this;
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public Builder clearIoDescriptor() {
-        if (ioDescriptorBuilder_ == null) {
-          ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
-          onChanged();
-        } else {
-          ioDescriptorBuilder_.clear();
-        }
-        bitField0_ = (bitField0_ & ~0x00000002);
-        return this;
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder getIoDescriptorBuilder() {
-        bitField0_ |= 0x00000002;
-        onChanged();
-        return getIoDescriptorFieldBuilder().getBuilder();
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
-        if (ioDescriptorBuilder_ != null) {
-          return ioDescriptorBuilder_.getMessageOrBuilder();
-        } else {
-          return ioDescriptor_;
-        }
-      }
-      /**
-       * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
-       */
-      private com.google.protobuf.SingleFieldBuilder<
-          org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder> 
-          getIoDescriptorFieldBuilder() {
-        if (ioDescriptorBuilder_ == null) {
-          ioDescriptorBuilder_ = new com.google.protobuf.SingleFieldBuilder<
-              org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder>(
-                  ioDescriptor_,
-                  getParentForChildren(),
-                  isClean());
-          ioDescriptor_ = null;
-        }
-        return ioDescriptorBuilder_;
-      }
-
-      // optional int32 physical_edge_count = 3;
-      private int physicalEdgeCount_ ;
-      /**
-       * <code>optional int32 physical_edge

<TRUNCATED>

[06/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
deleted file mode 100644
index 515bf3c..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-public class LlapNodeId {
-
-  private static final LoadingCache<LlapNodeId, LlapNodeId> CACHE =
-      CacheBuilder.newBuilder().softValues().build(
-          new CacheLoader<LlapNodeId, LlapNodeId>() {
-            @Override
-            public LlapNodeId load(LlapNodeId key) throws Exception {
-              return key;
-            }
-          });
-
-  public static LlapNodeId getInstance(String hostname, int port) {
-    return CACHE.getUnchecked(new LlapNodeId(hostname, port));
-  }
-
-
-  private final String hostname;
-  private final int port;
-
-
-  private LlapNodeId(String hostname, int port) {
-    this.hostname = hostname;
-    this.port = port;
-  }
-
-  public String getHostname() {
-    return hostname;
-  }
-
-  public int getPort() {
-    return port;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    LlapNodeId that = (LlapNodeId) o;
-
-    if (port != that.port) {
-      return false;
-    }
-    if (!hostname.equals(that.hostname)) {
-      return false;
-    }
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    int result = hostname.hashCode();
-    result = 1009 * result + port;
-    return result;
-  }
-
-  @Override
-  public String toString() {
-    return hostname + ":" + port;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 544af09..0399798 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -25,6 +25,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Collection;
 
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -217,7 +219,13 @@ public class LlapServiceDriver {
     CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), libDir.toString(), true);
     lfs.delete(new Path(libDir, "tez.tar.gz"), false);
 
+    // llap-common
+    lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapDaemonProtocolProtos.class)), libDir);
+    // llap-tez
+    lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapTezUtils.class)), libDir);
+    // llap-server
     lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapInputFormat.class)), libDir);
+    // hive-exec
     lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(HiveInputFormat.class)), libDir);
 
     // copy default aux classes (json/hbase)

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
deleted file mode 100644
index 4c09941..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon;
-
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.hive.llap.security.LlapTokenSelector;
-
-@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB", protocolVersion = 1)
-@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
-@TokenInfo(LlapTokenSelector.class)
-public interface LlapDaemonProtocolBlockingPB extends LlapDaemonProtocolProtos.LlapDaemonProtocol.BlockingInterface {
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java
deleted file mode 100644
index 4efadac..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.llap.daemon;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-
-@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB", protocolVersion = 1)
-@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
-public interface LlapManagementProtocolBlockingPB extends LlapDaemonProtocolProtos.LlapManagementProtocol.BlockingInterface {
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 98951e1..37120d2 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -65,7 +65,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
   private static final Logger LOG = LoggerFactory.getLogger(LlapDaemon.class);
 
   private final Configuration shuffleHandlerConf;
-  private final LlapDaemonProtocolServerImpl server;
+  private final LlapProtocolServerImpl server;
   private final ContainerRunnerImpl containerRunner;
   private final AMReporter amReporter;
   private final LlapRegistryService registry;
@@ -164,7 +164,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
 
     this.amReporter = new AMReporter(srvAddress, new QueryFailedHandlerProxy(), daemonConf);
 
-    this.server = new LlapDaemonProtocolServerImpl(
+    this.server = new LlapProtocolServerImpl(
         numHandlers, this, srvAddress, mngAddress, srvPort, mngPort);
 
     this.containerRunner = new ContainerRunnerImpl(daemonConf,

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
deleted file mode 100644
index 9c7d2e2..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon.impl;
-
-import javax.annotation.Nullable;
-import javax.net.SocketFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.ProtocolProxy;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.security.UserGroupInformation;
-
-// TODO Change all this to be based on a regular interface instead of relying on the Proto service - Exception signatures cannot be controlled without this for the moment.
-
-
-public class LlapDaemonProtocolClientImpl implements LlapDaemonProtocolBlockingPB {
-
-  private final Configuration conf;
-  private final InetSocketAddress serverAddr;
-  private final RetryPolicy retryPolicy;
-  private final SocketFactory socketFactory;
-  LlapDaemonProtocolBlockingPB proxy;
-
-
-  public LlapDaemonProtocolClientImpl(Configuration conf, String hostname, int port,
-                                      @Nullable RetryPolicy retryPolicy,
-                                      @Nullable SocketFactory socketFactory) {
-    this.conf = conf;
-    this.serverAddr = NetUtils.createSocketAddr(hostname, port);
-    this.retryPolicy = retryPolicy;
-    if (socketFactory == null) {
-      this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
-    } else {
-      this.socketFactory = socketFactory;
-    }
-  }
-
-  @Override
-  public SubmitWorkResponseProto submitWork(RpcController controller,
-                                                                     SubmitWorkRequestProto request) throws
-      ServiceException {
-    try {
-      return getProxy().submitWork(null, request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
-                                                            SourceStateUpdatedRequestProto request) throws
-      ServiceException {
-    try {
-      return getProxy().sourceStateUpdated(null, request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public QueryCompleteResponseProto queryComplete(RpcController controller,
-                                                  QueryCompleteRequestProto request) throws
-      ServiceException {
-    try {
-      return getProxy().queryComplete(null, request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public TerminateFragmentResponseProto terminateFragment(
-      RpcController controller,
-      TerminateFragmentRequestProto request) throws ServiceException {
-    try {
-      return getProxy().terminateFragment(null, request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  public LlapDaemonProtocolBlockingPB getProxy() throws IOException {
-    if (proxy == null) {
-      proxy = createProxy();
-    }
-    return proxy;
-  }
-
-  public LlapDaemonProtocolBlockingPB createProxy() throws IOException {
-    RPC.setProtocolEngine(conf, LlapDaemonProtocolBlockingPB.class, ProtobufRpcEngine.class);
-    ProtocolProxy<LlapDaemonProtocolBlockingPB> proxy =
-        RPC.getProtocolProxy(LlapDaemonProtocolBlockingPB.class, 0, serverAddr,
-            UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
-            retryPolicy);
-    return proxy.getProxy();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
deleted file mode 100644
index 45ca906..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon.impl;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedAction;
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.ByteArrayDataOutput;
-import com.google.common.io.ByteStreams;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.hive.llap.security.LlapSecurityHelper;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.hive.llap.security.SecretManager;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.security.LlapDaemonPolicyProvider;
-
-public class LlapDaemonProtocolServerImpl extends AbstractService
-    implements LlapDaemonProtocolBlockingPB, LlapManagementProtocolBlockingPB {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LlapDaemonProtocolServerImpl.class);
-
-  private final int numHandlers;
-  private final ContainerRunner containerRunner;
-  private final int srvPort, mngPort;
-  private RPC.Server server, mngServer;
-  private final AtomicReference<InetSocketAddress> srvAddress, mngAddress;
-  private SecretManager zkSecretManager;
-
-  public LlapDaemonProtocolServerImpl(int numHandlers,
-                                      ContainerRunner containerRunner,
-                                      AtomicReference<InetSocketAddress> srvAddress,
-                                      AtomicReference<InetSocketAddress> mngAddress,
-                                      int srvPort,
-                                      int mngPort) {
-    super("LlapDaemonProtocolServerImpl");
-    this.numHandlers = numHandlers;
-    this.containerRunner = containerRunner;
-    this.srvAddress = srvAddress;
-    this.srvPort = srvPort;
-    this.mngAddress = mngAddress;
-    this.mngPort = mngPort;
-    LOG.info("Creating: " + LlapDaemonProtocolServerImpl.class.getSimpleName() +
-        " with port configured to: " + srvPort);
-  }
-
-  @Override
-  public SubmitWorkResponseProto submitWork(RpcController controller,
-                                            SubmitWorkRequestProto request) throws
-      ServiceException {
-    try {
-      return containerRunner.submitWork(request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-
-  @Override
-  public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
-      SourceStateUpdatedRequestProto request) throws ServiceException {
-    return containerRunner.sourceStateUpdated(request);
-  }
-
-  @Override
-  public QueryCompleteResponseProto queryComplete(RpcController controller,
-      QueryCompleteRequestProto request) throws ServiceException {
-    return containerRunner.queryComplete(request);
-  }
-
-  @Override
-  public TerminateFragmentResponseProto terminateFragment(
-      RpcController controller,
-      TerminateFragmentRequestProto request) throws ServiceException {
-    return containerRunner.terminateFragment(request);
-  }
-
-  @Override
-  public void serviceStart() {
-    final Configuration conf = getConfig();
-    final BlockingService daemonImpl =
-        LlapDaemonProtocolProtos.LlapDaemonProtocol.newReflectiveBlockingService(this);
-    final BlockingService managementImpl =
-        LlapDaemonProtocolProtos.LlapManagementProtocol.newReflectiveBlockingService(this);
-    if (!UserGroupInformation.isSecurityEnabled()) {
-      startProtocolServers(conf, daemonImpl, managementImpl);
-      return;
-    }
-    String llapPrincipal = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_PRINCIPAL),
-        llapKeytab = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_KEYTAB_FILE);
-    zkSecretManager = SecretManager.createSecretManager(conf, llapPrincipal, llapKeytab);
-
-    // Start the protocol server after properly authenticating with daemon keytab.
-    UserGroupInformation daemonUgi = null;
-    try {
-      daemonUgi = LlapSecurityHelper.loginWithKerberos(llapPrincipal, llapKeytab);
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-    daemonUgi.doAs(new PrivilegedAction<Void>() {
-       @Override
-      public Void run() {
-         startProtocolServers(conf, daemonImpl, managementImpl);
-         return null;
-      }
-    });
-  }
-
-  private void startProtocolServers(
-      Configuration conf, BlockingService daemonImpl, BlockingService managementImpl) {
-    server = startProtocolServer(srvPort, numHandlers, srvAddress, conf, daemonImpl,
-        LlapDaemonProtocolBlockingPB.class);
-    mngServer = startProtocolServer(mngPort, 2, mngAddress, conf, managementImpl,
-        LlapManagementProtocolBlockingPB.class);
-  }
-
-  private RPC.Server startProtocolServer(int srvPort, int numHandlers,
-      AtomicReference<InetSocketAddress> bindAddress, Configuration conf,
-      BlockingService impl, Class<?> protocolClass) {
-    InetSocketAddress addr = new InetSocketAddress(srvPort);
-    RPC.Server server;
-    try {
-      server = createServer(protocolClass, addr, conf, numHandlers, impl);
-      server.start();
-    } catch (IOException e) {
-      LOG.error("Failed to run RPC Server on port: " + srvPort, e);
-      throw new RuntimeException(e);
-    }
-
-    InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server);
-    bindAddress.set(NetUtils.createSocketAddrForHost(
-        serverBindAddress.getAddress().getCanonicalHostName(),
-        serverBindAddress.getPort()));
-    LOG.info("Instantiated " + protocolClass.getSimpleName() + " at " + bindAddress);
-    return server;
-  }
-
-  @Override
-  public void serviceStop() {
-    if (server != null) {
-      server.stop();
-    }
-    if (mngServer != null) {
-      mngServer.stop();
-    }
-  }
-
-  @InterfaceAudience.Private
-  @VisibleForTesting
-  InetSocketAddress getBindAddress() {
-    return srvAddress.get();
-  }
-
-  private RPC.Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
-                                  int numHandlers, BlockingService blockingService) throws
-      IOException {
-    RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
-    RPC.Builder builder = new RPC.Builder(conf)
-        .setProtocol(pbProtocol)
-        .setInstance(blockingService)
-        .setBindAddress(addr.getHostName())
-        .setPort(addr.getPort())
-        .setNumHandlers(numHandlers);
-    if (zkSecretManager != null) {
-      builder = builder.setSecretManager(zkSecretManager);
-    }
-    RPC.Server server = builder.build();
-
-    if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
-      server.refreshServiceAcl(conf, new LlapDaemonPolicyProvider());
-    }
-    return server;
-  }
-
-
-  @Override
-  public GetTokenResponseProto getDelegationToken(RpcController controller,
-      GetTokenRequestProto request) throws ServiceException {
-    if (zkSecretManager == null) {
-      throw new ServiceException("Operation not supported on unsecure cluster");
-    }
-    UserGroupInformation ugi;
-    try {
-      ugi = UserGroupInformation.getCurrentUser();
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-    String user = ugi.getUserName();
-    Text owner = new Text(user);
-    Text realUser = null;
-    if (ugi.getRealUser() != null) {
-      realUser = new Text(ugi.getRealUser().getUserName());
-    }
-    Text renewer = new Text(ugi.getShortUserName());
-    LlapTokenIdentifier llapId = new LlapTokenIdentifier(owner, renewer, realUser);
-    // TODO: note that the token is not renewable right now and will last for 2 weeks by default.
-    Token<LlapTokenIdentifier> token = new Token<LlapTokenIdentifier>(llapId, zkSecretManager);
-    ByteArrayDataOutput out = ByteStreams.newDataOutput();
-    try {
-      token.write(out);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-    ByteString bs = ByteString.copyFrom(out.toByteArray());
-    GetTokenResponseProto response = GetTokenResponseProto.newBuilder().setToken(bs).build();
-    return response;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java
deleted file mode 100644
index e293a95..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon.impl;
-
-import javax.annotation.Nullable;
-import javax.net.SocketFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.ProtocolProxy;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
-import org.apache.hadoop.security.UserGroupInformation;
-
-public class LlapManagementProtocolClientImpl implements LlapManagementProtocolBlockingPB {
-
-  private final Configuration conf;
-  private final InetSocketAddress serverAddr;
-  private final RetryPolicy retryPolicy;
-  private final SocketFactory socketFactory;
-  LlapManagementProtocolBlockingPB proxy;
-
-
-  public LlapManagementProtocolClientImpl(Configuration conf, String hostname, int port,
-                                      @Nullable RetryPolicy retryPolicy,
-                                      @Nullable SocketFactory socketFactory) {
-    this.conf = conf;
-    this.serverAddr = NetUtils.createSocketAddr(hostname, port);
-    this.retryPolicy = retryPolicy;
-    if (socketFactory == null) {
-      this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
-    } else {
-      this.socketFactory = socketFactory;
-    }
-  }
-
-  public LlapManagementProtocolBlockingPB getProxy() throws IOException {
-    if (proxy == null) {
-      proxy = createProxy();
-    }
-    return proxy;
-  }
-
-  public LlapManagementProtocolBlockingPB createProxy() throws IOException {
-    RPC.setProtocolEngine(conf, LlapManagementProtocolBlockingPB.class, ProtobufRpcEngine.class);
-    ProtocolProxy<LlapManagementProtocolBlockingPB> proxy =
-        RPC.getProtocolProxy(LlapManagementProtocolBlockingPB.class, 0, serverAddr,
-            UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
-            retryPolicy);
-    return proxy.getProxy();
-  }
-
-  @Override
-  public GetTokenResponseProto getDelegationToken(RpcController controller,
-      GetTokenRequestProto request) throws ServiceException {
-    try {
-      return getProxy().getDelegationToken(null, request);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
new file mode 100644
index 0000000..c386d77
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.daemon.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.hive.llap.security.LlapSecurityHelper;
+import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
+import org.apache.hadoop.hive.llap.security.SecretManager;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
+import org.apache.hadoop.hive.llap.security.LlapDaemonPolicyProvider;
+
+public class LlapProtocolServerImpl extends AbstractService
+    implements LlapProtocolBlockingPB, LlapManagementProtocolPB {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LlapProtocolServerImpl.class);
+
+  private final int numHandlers;
+  private final ContainerRunner containerRunner;
+  private final int srvPort, mngPort;
+  private RPC.Server server, mngServer;
+  private final AtomicReference<InetSocketAddress> srvAddress, mngAddress;
+  private SecretManager zkSecretManager;
+
+  public LlapProtocolServerImpl(int numHandlers,
+                                ContainerRunner containerRunner,
+                                AtomicReference<InetSocketAddress> srvAddress,
+                                AtomicReference<InetSocketAddress> mngAddress,
+                                int srvPort,
+                                int mngPort) {
+    super("LlapDaemonProtocolServerImpl");
+    this.numHandlers = numHandlers;
+    this.containerRunner = containerRunner;
+    this.srvAddress = srvAddress;
+    this.srvPort = srvPort;
+    this.mngAddress = mngAddress;
+    this.mngPort = mngPort;
+    LOG.info("Creating: " + LlapProtocolServerImpl.class.getSimpleName() +
+        " with port configured to: " + srvPort);
+  }
+
+  @Override
+  public SubmitWorkResponseProto submitWork(RpcController controller,
+                                            SubmitWorkRequestProto request) throws
+      ServiceException {
+    try {
+      return containerRunner.submitWork(request);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
+      SourceStateUpdatedRequestProto request) throws ServiceException {
+    return containerRunner.sourceStateUpdated(request);
+  }
+
+  @Override
+  public QueryCompleteResponseProto queryComplete(RpcController controller,
+      QueryCompleteRequestProto request) throws ServiceException {
+    return containerRunner.queryComplete(request);
+  }
+
+  @Override
+  public TerminateFragmentResponseProto terminateFragment(
+      RpcController controller,
+      TerminateFragmentRequestProto request) throws ServiceException {
+    return containerRunner.terminateFragment(request);
+  }
+
+  @Override
+  public void serviceStart() {
+    final Configuration conf = getConfig();
+    final BlockingService daemonImpl =
+        LlapDaemonProtocolProtos.LlapDaemonProtocol.newReflectiveBlockingService(this);
+    final BlockingService managementImpl =
+        LlapDaemonProtocolProtos.LlapManagementProtocol.newReflectiveBlockingService(this);
+    if (!UserGroupInformation.isSecurityEnabled()) {
+      startProtocolServers(conf, daemonImpl, managementImpl);
+      return;
+    }
+    String llapPrincipal = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_PRINCIPAL),
+        llapKeytab = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_KEYTAB_FILE);
+    zkSecretManager = SecretManager.createSecretManager(conf, llapPrincipal, llapKeytab);
+
+    // Start the protocol server after properly authenticating with daemon keytab.
+    UserGroupInformation daemonUgi = null;
+    try {
+      daemonUgi = LlapSecurityHelper.loginWithKerberos(llapPrincipal, llapKeytab);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    daemonUgi.doAs(new PrivilegedAction<Void>() {
+       @Override
+      public Void run() {
+         startProtocolServers(conf, daemonImpl, managementImpl);
+         return null;
+      }
+    });
+  }
+
+  private void startProtocolServers(
+      Configuration conf, BlockingService daemonImpl, BlockingService managementImpl) {
+    server = startProtocolServer(srvPort, numHandlers, srvAddress, conf, daemonImpl,
+        LlapProtocolBlockingPB.class);
+    mngServer = startProtocolServer(mngPort, 2, mngAddress, conf, managementImpl,
+        LlapManagementProtocolPB.class);
+  }
+
+  private RPC.Server startProtocolServer(int srvPort, int numHandlers,
+      AtomicReference<InetSocketAddress> bindAddress, Configuration conf,
+      BlockingService impl, Class<?> protocolClass) {
+    InetSocketAddress addr = new InetSocketAddress(srvPort);
+    RPC.Server server;
+    try {
+      server = createServer(protocolClass, addr, conf, numHandlers, impl);
+      server.start();
+    } catch (IOException e) {
+      LOG.error("Failed to run RPC Server on port: " + srvPort, e);
+      throw new RuntimeException(e);
+    }
+
+    InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server);
+    bindAddress.set(NetUtils.createSocketAddrForHost(
+        serverBindAddress.getAddress().getCanonicalHostName(),
+        serverBindAddress.getPort()));
+    LOG.info("Instantiated " + protocolClass.getSimpleName() + " at " + bindAddress);
+    return server;
+  }
+
+  @Override
+  public void serviceStop() {
+    if (server != null) {
+      server.stop();
+    }
+    if (mngServer != null) {
+      mngServer.stop();
+    }
+  }
+
+  @InterfaceAudience.Private
+  @VisibleForTesting
+  InetSocketAddress getBindAddress() {
+    return srvAddress.get();
+  }
+
+  private RPC.Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
+                                  int numHandlers, BlockingService blockingService) throws
+      IOException {
+    RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
+    RPC.Builder builder = new RPC.Builder(conf)
+        .setProtocol(pbProtocol)
+        .setInstance(blockingService)
+        .setBindAddress(addr.getHostName())
+        .setPort(addr.getPort())
+        .setNumHandlers(numHandlers);
+    if (zkSecretManager != null) {
+      builder = builder.setSecretManager(zkSecretManager);
+    }
+    RPC.Server server = builder.build();
+
+    if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
+      server.refreshServiceAcl(conf, new LlapDaemonPolicyProvider());
+    }
+    return server;
+  }
+
+
+  @Override
+  public GetTokenResponseProto getDelegationToken(RpcController controller,
+      GetTokenRequestProto request) throws ServiceException {
+    if (zkSecretManager == null) {
+      throw new ServiceException("Operation not supported on unsecure cluster");
+    }
+    UserGroupInformation ugi;
+    try {
+      ugi = UserGroupInformation.getCurrentUser();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    String user = ugi.getUserName();
+    Text owner = new Text(user);
+    Text realUser = null;
+    if (ugi.getRealUser() != null) {
+      realUser = new Text(ugi.getRealUser().getUserName());
+    }
+    Text renewer = new Text(ugi.getShortUserName());
+    LlapTokenIdentifier llapId = new LlapTokenIdentifier(owner, renewer, realUser);
+    // TODO: note that the token is not renewable right now and will last for 2 weeks by default.
+    Token<LlapTokenIdentifier> token = new Token<LlapTokenIdentifier>(llapId, zkSecretManager);
+    ByteArrayDataOutput out = ByteStreams.newDataOutput();
+    try {
+      token.write(out);
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+    ByteString bs = ByteString.copyFrom(out.toByteArray());
+    GetTokenResponseProto response = GetTokenResponseProto.newBuilder().setToken(bs).build();
+    return response;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
index aa065a9..480a394 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.llap.daemon.FinishableStateUpdateHandler;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -89,7 +89,7 @@ public class QueryFragmentInfo {
     boolean canFinish = true;
     if (inputSpecList != null && !inputSpecList.isEmpty()) {
       for (IOSpecProto inputSpec : inputSpecList) {
-        if (isSourceOfInterest(inputSpec)) {
+        if (LlapTezUtils.isSourceOfInterest(inputSpec.getIoDescriptor().getClassName())) {
           // Lookup the state in the map.
           LlapDaemonProtocolProtos.SourceStateProto state = queryInfo.getSourceStateMap()
               .get(inputSpec.getConnectedVertexName());
@@ -129,7 +129,7 @@ public class QueryFragmentInfo {
     List<IOSpecProto> inputSpecList = fragmentSpec.getInputSpecsList();
     if (inputSpecList != null && !inputSpecList.isEmpty()) {
       for (IOSpecProto inputSpec : inputSpecList) {
-        if (isSourceOfInterest(inputSpec)) {
+        if (LlapTezUtils.isSourceOfInterest(inputSpec.getIoDescriptor().getClassName())) {
           sourcesOfInterest.add(inputSpec.getConnectedVertexName());
         }
       }
@@ -143,13 +143,6 @@ public class QueryFragmentInfo {
     queryInfo.unregisterFinishableStateUpdate(handler);
   }
 
-  private boolean isSourceOfInterest(IOSpecProto inputSpec) {
-    String inputClassName = inputSpec.getIoDescriptor().getClassName();
-    // MRInput is not of interest since it'll always be ready.
-    return !inputClassName.equals(MRInputLegacy.class.getName());
-  }
-
-
   @Override
   public boolean equals(Object o) {
     if (this == o) {

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
index ede2a03..d88d82a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecPro
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
 import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
-import org.apache.hadoop.hive.llap.tezplugins.Converters;
+import org.apache.hadoop.hive.llap.tez.Converters;
 import org.apache.hadoop.hive.ql.io.IOContextMap;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.net.NetUtils;

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java b/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
deleted file mode 100644
index 9549567..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
-import org.apache.tez.runtime.common.security.JobTokenSelector;
-
-@TokenInfo(JobTokenSelector.class)
-public interface LlapTaskUmbilicalProtocol extends VersionedProtocol {
-
-  public static final long versionID = 1L;
-
-  // From Tez. Eventually changes over to the LLAP protocol and ProtocolBuffers
-  boolean canCommit(TezTaskAttemptID taskid) throws IOException;
-  public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
-      throws IOException, TezException;
-
-  public void nodeHeartbeat(Text hostname, int port) throws IOException;
-
-  public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
index d67647b..bedd265 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
@@ -18,17 +18,17 @@
 package org.apache.hadoop.hive.llap.security;
 
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.security.authorize.Service;
 
 public class LlapDaemonPolicyProvider extends PolicyProvider {
   private static final Service[] services = new Service[] {
     new Service(HiveConf.ConfVars.LLAP_SECURITY_ACL.varname,
-        LlapDaemonProtocolBlockingPB.class),
+        LlapProtocolBlockingPB.class),
     new Service(HiveConf.ConfVars.LLAP_MANAGEMENT_ACL.varname,
-        LlapManagementProtocolBlockingPB.class)
+        LlapManagementProtocolPB.class)
   };
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
index a00b631..aa8745d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
 import javax.net.SocketFactory;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.llap.daemon.impl.LlapManagementProtocolClientImpl;
+import org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
 import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
 import org.apache.hadoop.hive.llap.registry.ServiceInstance;

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
index 4dca2ce..eb514f2 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
@@ -20,8 +20,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -36,8 +36,8 @@ public class LlapServerSecurityInfo extends SecurityInfo {
     if (LOG.isDebugEnabled()) {
       LOG.debug("Trying to get KerberosInfo for " + protocol);
     }
-    if (!LlapDaemonProtocolBlockingPB.class.isAssignableFrom(protocol)
-        && !LlapManagementProtocolBlockingPB.class.isAssignableFrom(protocol)) return null;
+    if (!LlapProtocolBlockingPB.class.isAssignableFrom(protocol)
+        && !LlapManagementProtocolPB.class.isAssignableFrom(protocol)) return null;
     return new KerberosInfo() {
       @Override
       public Class<? extends Annotation> annotationType() {
@@ -62,7 +62,7 @@ public class LlapServerSecurityInfo extends SecurityInfo {
       LOG.debug("Trying to get TokenInfo for " + protocol);
     }
     // Tokens cannot be used for the management protocol (for now).
-    if (!LlapDaemonProtocolBlockingPB.class.isAssignableFrom(protocol)) return null;
+    if (!LlapProtocolBlockingPB.class.isAssignableFrom(protocol)) return null;
     return new TokenInfo() {
       @Override
       public Class<? extends Annotation> annotationType() {

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
deleted file mode 100644
index b6e7499..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import java.util.Collection;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
-
-public class LlapTokenSelector implements TokenSelector<LlapTokenIdentifier> {
-  private static final Log LOG = LogFactory.getLog(LlapTokenSelector.class);
-
-  @Override
-  public Token<LlapTokenIdentifier> selectToken(Text service,
-      Collection<Token<? extends TokenIdentifier>> tokens) {
-    if (service == null) return null;
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Looking for a token with service " + service);
-    }
-    for (Token<? extends TokenIdentifier> token : tokens) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Token = " + token.getKind() + "; service = " + token.getService());
-      }
-      if (LlapTokenIdentifier.KIND_NAME.equals(token.getKind())
-          && service.equals(token.getService())) {
-        @SuppressWarnings("unchecked")
-        Token<LlapTokenIdentifier> result = (Token<LlapTokenIdentifier>)token;
-        return result;
-      }
-    }
-    return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java
deleted file mode 100644
index f61d62f..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GroupInputSpecProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto;
-import org.apache.tez.common.TezCommonUtils;
-import org.apache.tez.dag.api.EntityDescriptor;
-import org.apache.tez.dag.api.InputDescriptor;
-import org.apache.tez.dag.api.OutputDescriptor;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.event.VertexState;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.GroupInputSpec;
-import org.apache.tez.runtime.api.impl.InputSpec;
-import org.apache.tez.runtime.api.impl.OutputSpec;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-
-public class Converters {
-
-  public static TaskSpec getTaskSpecfromProto(FragmentSpecProto FragmentSpecProto) {
-    TezTaskAttemptID taskAttemptID =
-        TezTaskAttemptID.fromString(FragmentSpecProto.getFragmentIdentifierString());
-
-    ProcessorDescriptor processorDescriptor = null;
-    if (FragmentSpecProto.hasProcessorDescriptor()) {
-      processorDescriptor = convertProcessorDescriptorFromProto(
-          FragmentSpecProto.getProcessorDescriptor());
-    }
-
-    List<InputSpec> inputSpecList = new ArrayList<InputSpec>(FragmentSpecProto.getInputSpecsCount());
-    if (FragmentSpecProto.getInputSpecsCount() > 0) {
-      for (IOSpecProto inputSpecProto : FragmentSpecProto.getInputSpecsList()) {
-        inputSpecList.add(getInputSpecFromProto(inputSpecProto));
-      }
-    }
-
-    List<OutputSpec> outputSpecList =
-        new ArrayList<OutputSpec>(FragmentSpecProto.getOutputSpecsCount());
-    if (FragmentSpecProto.getOutputSpecsCount() > 0) {
-      for (IOSpecProto outputSpecProto : FragmentSpecProto.getOutputSpecsList()) {
-        outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
-      }
-    }
-
-    List<GroupInputSpec> groupInputSpecs =
-        new ArrayList<GroupInputSpec>(FragmentSpecProto.getGroupedInputSpecsCount());
-    if (FragmentSpecProto.getGroupedInputSpecsCount() > 0) {
-      for (GroupInputSpecProto groupInputSpecProto : FragmentSpecProto.getGroupedInputSpecsList()) {
-        groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
-      }
-    }
-
-    TaskSpec taskSpec =
-        new TaskSpec(taskAttemptID, FragmentSpecProto.getDagName(), FragmentSpecProto.getVertexName(),
-            FragmentSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList,
-            outputSpecList, groupInputSpecs);
-    return taskSpec;
-  }
-
-  public static FragmentSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
-    FragmentSpecProto.Builder builder = FragmentSpecProto.newBuilder();
-    builder.setFragmentIdentifierString(taskSpec.getTaskAttemptID().toString());
-    builder.setDagName(taskSpec.getDAGName());
-    builder.setVertexName(taskSpec.getVertexName());
-    builder.setVertexParallelism(taskSpec.getVertexParallelism());
-    builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId());
-    builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId());
-
-    if (taskSpec.getProcessorDescriptor() != null) {
-      builder.setProcessorDescriptor(
-          convertToProto(taskSpec.getProcessorDescriptor()));
-    }
-
-    if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
-      for (InputSpec inputSpec : taskSpec.getInputs()) {
-        builder.addInputSpecs(convertInputSpecToProto(inputSpec));
-      }
-    }
-
-    if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
-      for (OutputSpec outputSpec : taskSpec.getOutputs()) {
-        builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
-      }
-    }
-
-    if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
-      for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
-        builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
-
-      }
-    }
-    return builder.build();
-  }
-
-  private static ProcessorDescriptor convertProcessorDescriptorFromProto(
-      EntityDescriptorProto proto) {
-    String className = proto.getClassName();
-    UserPayload payload = convertPayloadFromProto(proto);
-    ProcessorDescriptor pd = ProcessorDescriptor.create(className);
-    setUserPayload(pd, payload);
-    return pd;
-  }
-
-  private static EntityDescriptorProto convertToProto(
-      EntityDescriptor<?> descriptor) {
-    EntityDescriptorProto.Builder builder = EntityDescriptorProto
-        .newBuilder();
-    builder.setClassName(descriptor.getClassName());
-
-    UserPayload userPayload = descriptor.getUserPayload();
-    if (userPayload != null) {
-      UserPayloadProto.Builder payloadBuilder = UserPayloadProto.newBuilder();
-      if (userPayload.hasPayload()) {
-        payloadBuilder.setUserPayload(ByteString.copyFrom(userPayload.getPayload()));
-        payloadBuilder.setVersion(userPayload.getVersion());
-      }
-      builder.setUserPayload(payloadBuilder.build());
-    }
-    if (descriptor.getHistoryText() != null) {
-      try {
-        builder.setHistoryText(TezCommonUtils.compressByteArrayToByteString(
-            descriptor.getHistoryText().getBytes("UTF-8")));
-      } catch (IOException e) {
-        throw new TezUncheckedException(e);
-      }
-    }
-    return builder.build();
-  }
-
-  private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
-    InputDescriptor inputDescriptor = null;
-    if (inputSpecProto.hasIoDescriptor()) {
-      inputDescriptor =
-          convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor());
-    }
-    InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
-        inputSpecProto.getPhysicalEdgeCount());
-    return inputSpec;
-  }
-
-  private static InputDescriptor convertInputDescriptorFromProto(
-      EntityDescriptorProto proto) {
-    String className = proto.getClassName();
-    UserPayload payload = convertPayloadFromProto(proto);
-    InputDescriptor id = InputDescriptor.create(className);
-    setUserPayload(id, payload);
-    return id;
-  }
-
-  private static OutputDescriptor convertOutputDescriptorFromProto(
-      EntityDescriptorProto proto) {
-    String className = proto.getClassName();
-    UserPayload payload = convertPayloadFromProto(proto);
-    OutputDescriptor od = OutputDescriptor.create(className);
-    setUserPayload(od, payload);
-    return od;
-  }
-
-  private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
-    IOSpecProto.Builder builder = IOSpecProto.newBuilder();
-    if (inputSpec.getSourceVertexName() != null) {
-      builder.setConnectedVertexName(inputSpec.getSourceVertexName());
-    }
-    if (inputSpec.getInputDescriptor() != null) {
-      builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor()));
-    }
-    builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
-    return builder.build();
-  }
-
-  private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
-    OutputDescriptor outputDescriptor = null;
-    if (outputSpecProto.hasIoDescriptor()) {
-      outputDescriptor =
-          convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor());
-    }
-    OutputSpec outputSpec =
-        new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
-            outputSpecProto.getPhysicalEdgeCount());
-    return outputSpec;
-  }
-
-  public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
-    IOSpecProto.Builder builder = IOSpecProto.newBuilder();
-    if (outputSpec.getDestinationVertexName() != null) {
-      builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
-    }
-    if (outputSpec.getOutputDescriptor() != null) {
-      builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor()));
-    }
-    builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
-    return builder.build();
-  }
-
-  private static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
-    GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
-        groupInputSpecProto.getGroupVerticesList(),
-        convertInputDescriptorFromProto(groupInputSpecProto.getMergedInputDescriptor()));
-    return groupSpec;
-  }
-
-  private static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
-    GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
-    builder.setGroupName(groupInputSpec.getGroupName());
-    builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
-    builder.setMergedInputDescriptor(convertToProto(groupInputSpec.getMergedInputDescriptor()));
-    return builder.build();
-  }
-
-
-  private static void setUserPayload(EntityDescriptor<?> entity, UserPayload payload) {
-    if (payload != null) {
-      entity.setUserPayload(payload);
-    }
-  }
-
-  private static UserPayload convertPayloadFromProto(
-      EntityDescriptorProto proto) {
-    UserPayload userPayload = null;
-    if (proto.hasUserPayload()) {
-      if (proto.getUserPayload().hasUserPayload()) {
-        userPayload =
-            UserPayload.create(proto.getUserPayload().getUserPayload().asReadOnlyByteBuffer(), proto.getUserPayload().getVersion());
-      } else {
-        userPayload = UserPayload.create(null);
-      }
-    }
-    return userPayload;
-  }
-
-  public static SourceStateProto fromVertexState(VertexState state) {
-    switch (state) {
-      case SUCCEEDED:
-        return SourceStateProto.S_SUCCEEDED;
-      case RUNNING:
-        return SourceStateProto.S_RUNNING;
-      default:
-        throw new RuntimeException("Unexpected state: " + state);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
deleted file mode 100644
index 07703a2..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapContainerLauncher extends ContainerLauncher {
-  private static final Logger LOG = LoggerFactory.getLogger(LlapContainerLauncher.class);
-
-  public LlapContainerLauncher(ContainerLauncherContext containerLauncherContext) {
-    super(containerLauncherContext);
-  }
-
-  @Override
-  public void launchContainer(ContainerLaunchRequest containerLaunchRequest) {
-    LOG.info("No-op launch for container: " + containerLaunchRequest.getContainerId() +
-        " succeeded on host: " + containerLaunchRequest.getNodeId());
-    getContext().containerLaunched(containerLaunchRequest.getContainerId());
-  }
-
-  @Override
-  public void stopContainer(ContainerStopRequest containerStopRequest) {
-    LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + containerStopRequest);
-    getContext().containerStopRequested(containerStopRequest.getContainerId());
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java
deleted file mode 100644
index 2884e40..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import javax.net.SocketFactory;
-
-import java.io.IOException;
-import java.security.PrivilegedAction;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.Message;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemonProtocolClientImpl;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.service.AbstractService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapDaemonProtocolClientProxy extends AbstractService {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LlapDaemonProtocolClientProxy.class);
-
-  private final ConcurrentMap<String, LlapDaemonProtocolBlockingPB> hostProxies;
-
-  private final RequestManager requestManager;
-  private final RetryPolicy retryPolicy;
-  private final SocketFactory socketFactory;
-
-  private final ListeningExecutorService requestManagerExecutor;
-  private volatile ListenableFuture<Void> requestManagerFuture;
-  private final Token<LlapTokenIdentifier> llapToken;
-
-  public LlapDaemonProtocolClientProxy(
-      int numThreads, Configuration conf, Token<LlapTokenIdentifier> llapToken) {
-    super(LlapDaemonProtocolClientProxy.class.getSimpleName());
-    this.hostProxies = new ConcurrentHashMap<>();
-    this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
-    this.llapToken = llapToken;
-
-    long connectionTimeout = HiveConf.getTimeVar(conf,
-        ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS);
-    long retrySleep = HiveConf.getTimeVar(conf,
-        ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MS,
-        TimeUnit.MILLISECONDS);
-    this.retryPolicy = RetryPolicies.retryUpToMaximumTimeWithFixedSleep(
-        connectionTimeout, retrySleep, TimeUnit.MILLISECONDS);
-
-    this.requestManager = new RequestManager(numThreads);
-    ExecutorService localExecutor = Executors.newFixedThreadPool(1,
-        new ThreadFactoryBuilder().setNameFormat("RequestManagerExecutor").build());
-    this.requestManagerExecutor = MoreExecutors.listeningDecorator(localExecutor);
-
-    LOG.info("Setting up taskCommunicator with" +
-        "numThreads=" + numThreads +
-        "retryTime(millis)=" + connectionTimeout +
-        "retrySleep(millis)=" + retrySleep);
-  }
-
-  @Override
-  public void serviceStart() {
-    requestManagerFuture = requestManagerExecutor.submit(requestManager);
-    Futures.addCallback(requestManagerFuture, new FutureCallback<Void>() {
-      @Override
-      public void onSuccess(Void result) {
-        LOG.info("RequestManager shutdown");
-      }
-
-      @Override
-      public void onFailure(Throwable t) {
-        LOG.warn("RequestManager shutdown with error", t);
-      }
-    });
-  }
-
-  @Override
-  public void serviceStop() {
-    if (requestManagerFuture != null) {
-      requestManager.shutdown();
-      requestManagerFuture.cancel(true);
-    }
-    requestManagerExecutor.shutdown();
-  }
-
-  public void sendSubmitWork(SubmitWorkRequestProto request, String host, int port,
-                         final ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
-    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
-    requestManager.queueRequest(new SubmitWorkCallable(nodeId, request, callback));
-  }
-
-  public void sendSourceStateUpdate(final SourceStateUpdatedRequestProto request, final String host,
-                                    final int port,
-                                    final ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
-    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
-    requestManager.queueRequest(
-        new SendSourceStateUpdateCallable(nodeId, request, callback));
-  }
-
-  public void sendQueryComplete(final QueryCompleteRequestProto request, final String host,
-                                final int port,
-                                final ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
-    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
-    requestManager.queueRequest(new SendQueryCompleteCallable(nodeId, request, callback));
-  }
-
-  public void sendTerminateFragment(final TerminateFragmentRequestProto request, final String host,
-                                    final int port,
-                                    final ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
-    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
-    requestManager.queueRequest(new SendTerminateFragmentCallable(nodeId, request, callback));
-  }
-
-  @VisibleForTesting
-  static class RequestManager implements Callable<Void> {
-
-    private final Lock lock = new ReentrantLock();
-    private final AtomicBoolean isShutdown = new AtomicBoolean(false);
-    private final Condition queueCondition = lock.newCondition();
-    private final AtomicBoolean shouldRun = new AtomicBoolean(false);
-
-    private final int maxConcurrentRequestsPerNode = 1;
-    private final ListeningExecutorService executor;
-
-
-    // Tracks new additions via add, while the loop is processing existing ones.
-    private final LinkedList<CallableRequest> newRequestList = new LinkedList<>();
-
-    // Tracks existing requests which are cycled through.
-    private final LinkedList<CallableRequest> pendingRequests = new LinkedList<>();
-
-    // Tracks requests executing per node
-    private final ConcurrentMap<LlapNodeId, AtomicInteger> runningRequests = new ConcurrentHashMap<>();
-
-    // Tracks completed requests pre node
-    private final LinkedList<LlapNodeId> completedNodes = new LinkedList<>();
-
-    public RequestManager(int numThreads) {
-      ExecutorService localExecutor = Executors.newFixedThreadPool(numThreads,
-          new ThreadFactoryBuilder().setNameFormat("TaskCommunicator #%2d").build());
-      executor = MoreExecutors.listeningDecorator(localExecutor);
-    }
-
-
-    @VisibleForTesting
-    Set<LlapNodeId> currentLoopDisabledNodes = new HashSet<>();
-    @VisibleForTesting
-    List<CallableRequest> currentLoopSkippedRequests = new LinkedList<>();
-    @Override
-    public Void call() {
-      // Caches disabled nodes for quicker lookups and ensures a request on a node which was skipped
-      // does not go out of order.
-      while (!isShutdown.get()) {
-        lock.lock();
-        try {
-          while (!shouldRun.get()) {
-            queueCondition.await();
-            break; // Break out and try executing.
-          }
-          boolean shouldBreak = process();
-          if (shouldBreak) {
-            break;
-          }
-        } catch (InterruptedException e) {
-          if (isShutdown.get()) {
-            break;
-          } else {
-            LOG.warn("RunLoop interrupted without being shutdown first");
-            throw new RuntimeException(e);
-          }
-        } finally {
-          lock.unlock();
-        }
-      }
-      LOG.info("CallScheduler loop exiting");
-      return null;
-    }
-
-    /* Add a new request to be executed */
-    public void queueRequest(CallableRequest request) {
-      synchronized (newRequestList) {
-        newRequestList.add(request);
-        shouldRun.set(true);
-      }
-      notifyRunLoop();
-    }
-
-    /* Indicates a request has completed on a node */
-    public void requestFinished(LlapNodeId nodeId) {
-      synchronized (completedNodes) {
-        completedNodes.add(nodeId);
-        shouldRun.set(true);
-      }
-      notifyRunLoop();
-    }
-
-    public void shutdown() {
-      if (!isShutdown.getAndSet(true)) {
-        executor.shutdownNow();
-        notifyRunLoop();
-      }
-    }
-
-    @VisibleForTesting
-    void submitToExecutor(CallableRequest request, LlapNodeId nodeId) {
-      ListenableFuture<SourceStateUpdatedResponseProto> future =
-          executor.submit(request);
-      Futures.addCallback(future, new ResponseCallback(request.getCallback(), nodeId, this));
-    }
-
-    @VisibleForTesting
-    boolean process() {
-      if (isShutdown.get()) {
-        return true;
-      }
-      currentLoopDisabledNodes.clear();
-      currentLoopSkippedRequests.clear();
-
-      // Set to false to block the next loop. This must be called before draining the lists,
-      // otherwise an add/completion after draining the lists but before setting it to false,
-      // will not trigger a run. May cause one unnecessary run if an add comes in before drain.
-      // drain list. add request (setTrue). setFalse needs to be avoided.
-      shouldRun.compareAndSet(true, false);
-      // Drain any calls which may have come in during the last execution of the loop.
-      drainNewRequestList();  // Locks newRequestList
-      drainCompletedNodes();  // Locks completedNodes
-
-
-      Iterator<CallableRequest> iterator = pendingRequests.iterator();
-      while (iterator.hasNext()) {
-        CallableRequest request = iterator.next();
-        iterator.remove();
-        LlapNodeId nodeId = request.getNodeId();
-        if (canRunForNode(nodeId, currentLoopDisabledNodes)) {
-          submitToExecutor(request, nodeId);
-        } else {
-          currentLoopDisabledNodes.add(nodeId);
-          currentLoopSkippedRequests.add(request);
-        }
-      }
-      // Tried scheduling everything that could be scheduled in this loop.
-      pendingRequests.addAll(0, currentLoopSkippedRequests);
-      return false;
-    }
-
-    private void drainNewRequestList() {
-      synchronized (newRequestList) {
-        if (!newRequestList.isEmpty()) {
-          pendingRequests.addAll(newRequestList);
-          newRequestList.clear();
-        }
-      }
-    }
-
-    private void drainCompletedNodes() {
-      synchronized (completedNodes) {
-        if (!completedNodes.isEmpty()) {
-          for (LlapNodeId nodeId : completedNodes) {
-            runningRequests.get(nodeId).decrementAndGet();
-          }
-        }
-        completedNodes.clear();
-      }
-    }
-
-    private boolean canRunForNode(LlapNodeId nodeId, Set<LlapNodeId> currentRunDisabledNodes) {
-      if (currentRunDisabledNodes.contains(nodeId)) {
-        return false;
-      } else {
-        AtomicInteger count = runningRequests.get(nodeId);
-        if (count == null) {
-          count = new AtomicInteger(0);
-          AtomicInteger old = runningRequests.putIfAbsent(nodeId, count);
-          count = old != null ? old : count;
-        }
-        if (count.incrementAndGet() <= maxConcurrentRequestsPerNode) {
-          return true;
-        } else {
-          count.decrementAndGet();
-          return false;
-        }
-      }
-    }
-
-    private void notifyRunLoop() {
-      lock.lock();
-      try {
-        queueCondition.signal();
-      } finally {
-        lock.unlock();
-      }
-    }
-  }
-
-
-  private static final class ResponseCallback<TYPE extends Message>
-      implements FutureCallback<TYPE> {
-
-    private final ExecuteRequestCallback<TYPE> callback;
-    private final LlapNodeId nodeId;
-    private final RequestManager requestManager;
-
-    public ResponseCallback(ExecuteRequestCallback<TYPE> callback, LlapNodeId nodeId,
-                            RequestManager requestManager) {
-      this.callback = callback;
-      this.nodeId = nodeId;
-      this.requestManager = requestManager;
-    }
-
-    @Override
-    public void onSuccess(TYPE result) {
-      try {
-        callback.setResponse(result);
-      } finally {
-        requestManager.requestFinished(nodeId);
-      }
-    }
-
-    @Override
-    public void onFailure(Throwable t) {
-      try {
-        callback.indicateError(t);
-      } finally {
-        requestManager.requestFinished(nodeId);
-      }
-    }
-  }
-
-  @VisibleForTesting
-  static abstract class CallableRequest<REQUEST extends Message, RESPONSE extends Message>
-      implements Callable {
-
-    final LlapNodeId nodeId;
-    final ExecuteRequestCallback<RESPONSE> callback;
-    final REQUEST request;
-
-
-    protected CallableRequest(LlapNodeId nodeId, REQUEST request, ExecuteRequestCallback<RESPONSE> callback) {
-      this.nodeId = nodeId;
-      this.request = request;
-      this.callback = callback;
-    }
-
-    public LlapNodeId getNodeId() {
-      return nodeId;
-    }
-
-    public ExecuteRequestCallback<RESPONSE> getCallback() {
-      return callback;
-    }
-
-    public abstract RESPONSE call() throws Exception;
-  }
-
-  private class SubmitWorkCallable extends CallableRequest<SubmitWorkRequestProto, SubmitWorkResponseProto> {
-
-    protected SubmitWorkCallable(LlapNodeId nodeId,
-                          SubmitWorkRequestProto submitWorkRequestProto,
-                                 ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
-      super(nodeId, submitWorkRequestProto, callback);
-    }
-
-    @Override
-    public SubmitWorkResponseProto call() throws Exception {
-      return getProxy(nodeId).submitWork(null, request);
-    }
-  }
-
-  private class SendSourceStateUpdateCallable
-      extends CallableRequest<SourceStateUpdatedRequestProto, SourceStateUpdatedResponseProto> {
-
-    public SendSourceStateUpdateCallable(LlapNodeId nodeId,
-                                         SourceStateUpdatedRequestProto request,
-                                         ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
-      super(nodeId, request, callback);
-    }
-
-    @Override
-    public SourceStateUpdatedResponseProto call() throws Exception {
-      return getProxy(nodeId).sourceStateUpdated(null, request);
-    }
-  }
-
-  private class SendQueryCompleteCallable
-      extends CallableRequest<QueryCompleteRequestProto, QueryCompleteResponseProto> {
-
-    protected SendQueryCompleteCallable(LlapNodeId nodeId,
-                                        QueryCompleteRequestProto queryCompleteRequestProto,
-                                        ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
-      super(nodeId, queryCompleteRequestProto, callback);
-    }
-
-    @Override
-    public QueryCompleteResponseProto call() throws Exception {
-      return getProxy(nodeId).queryComplete(null, request);
-    }
-  }
-
-  private class SendTerminateFragmentCallable
-      extends CallableRequest<TerminateFragmentRequestProto, TerminateFragmentResponseProto> {
-
-    protected SendTerminateFragmentCallable(LlapNodeId nodeId,
-                                            TerminateFragmentRequestProto terminateFragmentRequestProto,
-                                            ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
-      super(nodeId, terminateFragmentRequestProto, callback);
-    }
-
-    @Override
-    public TerminateFragmentResponseProto call() throws Exception {
-      return getProxy(nodeId).terminateFragment(null, request);
-    }
-  }
-
-  public interface ExecuteRequestCallback<T extends Message> {
-    void setResponse(T response);
-    void indicateError(Throwable t);
-  }
-
-  private LlapDaemonProtocolBlockingPB getProxy(final LlapNodeId nodeId) {
-    String hostId = getHostIdentifier(nodeId.getHostname(), nodeId.getPort());
-
-    LlapDaemonProtocolBlockingPB proxy = hostProxies.get(hostId);
-    if (proxy == null) {
-      if (llapToken == null) {
-        proxy = new LlapDaemonProtocolClientImpl(getConfig(), nodeId.getHostname(),
-            nodeId.getPort(), retryPolicy, socketFactory);
-      } else {
-        UserGroupInformation ugi;
-        try {
-          ugi = UserGroupInformation.getCurrentUser();
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-        Token<LlapTokenIdentifier> nodeToken = new Token<LlapTokenIdentifier>(llapToken);
-        SecurityUtil.setTokenService(nodeToken, NetUtils.createSocketAddrForHost(
-            nodeId.getHostname(), nodeId.getPort()));
-        ugi.addToken(nodeToken);
-        proxy = ugi.doAs(new PrivilegedAction<LlapDaemonProtocolBlockingPB>() {
-          @Override
-          public LlapDaemonProtocolBlockingPB run() {
-           return new LlapDaemonProtocolClientImpl(getConfig(), nodeId.getHostname(),
-               nodeId.getPort(), retryPolicy, socketFactory);
-          }
-        });
-      }
-
-      LlapDaemonProtocolBlockingPB proxyOld = hostProxies.putIfAbsent(hostId, proxy);
-      if (proxyOld != null) {
-        // TODO Shutdown the new proxy.
-        proxy = proxyOld;
-      }
-    }
-    return proxy;
-  }
-
-  private String getHostIdentifier(String hostname, int port) {
-    return hostname + ":" + port;
-  }
-}


[04/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
deleted file mode 100644
index 6beb4f8..0000000
--- a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
+++ /dev/null
@@ -1,1512 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.NavigableMap;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import org.apache.commons.lang.mutable.MutableInt;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.registry.ServiceInstance;
-import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
-import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Clock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskScheduler;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapTaskSchedulerService extends TaskScheduler {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LlapTaskSchedulerService.class);
-
-  private final Configuration conf;
-
-  // interface into the registry service
-  private ServiceInstanceSet activeInstances;
-
-  // Tracks all instances, including ones which have been disabled in the past.
-  // LinkedHashMap to provide the same iteration order when selecting a random host.
-  @VisibleForTesting
-  final Map<ServiceInstance, NodeInfo> instanceToNodeMap = new LinkedHashMap<>();
-  // TODO Ideally, remove elements from this once it's known that no tasks are linked to the instance (all deallocated)
-
-  // Tracks tasks which could not be allocated immediately.
-  @VisibleForTesting
-  final TreeMap<Priority, List<TaskInfo>> pendingTasks = new TreeMap<>(new Comparator<Priority>() {
-    @Override
-    public int compare(Priority o1, Priority o2) {
-      return o1.getPriority() - o2.getPriority();
-    }
-  });
-
-  // Tracks running and queued tasks. Cleared after a task completes.
-  private final ConcurrentMap<Object, TaskInfo> knownTasks = new ConcurrentHashMap<>();
-  private final TreeMap<Integer, TreeSet<TaskInfo>> runningTasks = new TreeMap<>();
-  private static final TaskStartComparator TASK_INFO_COMPARATOR = new TaskStartComparator();
-
-  // Queue for disabled nodes. Nodes make it out of this queue when their expiration timeout is hit.
-  @VisibleForTesting
-  final DelayQueue<NodeInfo> disabledNodesQueue = new DelayQueue<>();
-
-  private final boolean forceLocation;
-
-  private final ContainerFactory containerFactory;
-  private final Random random = new Random();
-  private final Clock clock;
-
-  private final ListeningExecutorService nodeEnabledExecutor;
-  private final NodeEnablerCallable nodeEnablerCallable =
-      new NodeEnablerCallable();
-
-  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-  private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
-  private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
-
-  private final Lock scheduleLock = new ReentrantLock();
-  private final Condition scheduleCondition = scheduleLock.newCondition();
-  private final AtomicBoolean pendingScheduleInvodations = new AtomicBoolean(false);
-  private final ListeningExecutorService schedulerExecutor;
-  private final SchedulerCallable schedulerCallable = new SchedulerCallable();
-
-  private final AtomicBoolean isStopped = new AtomicBoolean(false);
-  // Tracks total pending preemptions.
-  private final AtomicInteger pendingPreemptions = new AtomicInteger(0);
-  // Tracks pending preemptions per host, using the hostname || Always to be accessed inside a lock
-  private final Map<String, MutableInt> pendingPreemptionsPerHost = new HashMap<>();
-
-  private final NodeBlacklistConf nodeBlacklistConf;
-
-  // Per daemon
-  private final int memoryPerInstance;
-  private final int coresPerInstance;
-  private final int executorsPerInstance;
-
-  private final int numSchedulableTasksPerNode;
-
-  // Per Executor Thread
-  private final Resource resourcePerExecutor;
-
-  private final LlapRegistryService registry = new LlapRegistryService(false);
-
-  private volatile ListenableFuture<Void> nodeEnablerFuture;
-  private volatile ListenableFuture<Void> schedulerFuture;
-
-  @VisibleForTesting
-  private final AtomicInteger dagCounter = new AtomicInteger(1);
-  // Statistics to track allocations
-  // All of stats variables are visible for testing.
-  @VisibleForTesting
-  StatsPerDag dagStats = new StatsPerDag();
-
-  public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) {
-    this(taskSchedulerContext, new SystemClock());
-  }
-
-  @VisibleForTesting
-  public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock) {
-    super(taskSchedulerContext);
-    this.clock = clock;
-    try {
-      this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
-    } catch (IOException e) {
-      throw new TezUncheckedException(
-          "Failed to parse user payload for " + LlapTaskSchedulerService.class.getSimpleName(), e);
-    }
-    this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(),
-        taskSchedulerContext.getCustomClusterIdentifier());
-    this.memoryPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB);
-    this.coresPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE);
-    this.executorsPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
-    this.nodeBlacklistConf = new NodeBlacklistConf(
-        HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS,
-            TimeUnit.MILLISECONDS),
-        HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MAX_TIMEOUT_MS,
-            TimeUnit.MILLISECONDS),
-        HiveConf.getFloatVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_DISABLE_BACK_OFF_FACTOR));
-
-    this.numSchedulableTasksPerNode =
-        HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NUM_SCHEDULABLE_TASKS_PER_NODE);
-
-    long localityDelayMs = HiveConf
-        .getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY, TimeUnit.MILLISECONDS);
-    if (localityDelayMs == -1) {
-      this.forceLocation = true;
-    } else {
-      this.forceLocation = false;
-    }
-
-    int memoryPerExecutor = (int) (memoryPerInstance / (float) executorsPerInstance);
-    int coresPerExecutor = (int) (coresPerInstance / (float) executorsPerInstance);
-    this.resourcePerExecutor = Resource.newInstance(memoryPerExecutor, coresPerExecutor);
-
-    String instanceId = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
-
-    Preconditions.checkNotNull(instanceId, ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname
-        + " must be defined");
-
-    ExecutorService executorServiceRaw =
-        Executors.newFixedThreadPool(1,
-            new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapSchedulerNodeEnabler").build());
-    nodeEnabledExecutor = MoreExecutors.listeningDecorator(executorServiceRaw);
-
-    ExecutorService schedulerExecutorServiceRaw = Executors.newFixedThreadPool(1,
-        new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build());
-    schedulerExecutor = MoreExecutors.listeningDecorator(schedulerExecutorServiceRaw);
-
-    LOG.info("Running with configuration: " + "memoryPerInstance=" + memoryPerInstance
-        + ", vCoresPerInstance=" + coresPerInstance + ", executorsPerInstance="
-        + executorsPerInstance + ", resourcePerInstanceInferred=" + resourcePerExecutor
-        + ", nodeBlacklistConf=" + nodeBlacklistConf
-        + ", forceLocation=" + forceLocation);
-  }
-
-  @Override
-  public void initialize() {
-    registry.init(conf);
-  }
-
-  @Override
-  public void start() throws IOException {
-    writeLock.lock();
-    try {
-      nodeEnablerFuture = nodeEnabledExecutor.submit(nodeEnablerCallable);
-      Futures.addCallback(nodeEnablerFuture, new FutureCallback<Void>() {
-        @Override
-        public void onSuccess(Void result) {
-          LOG.info("NodeEnabledThread exited");
-        }
-
-        @Override
-        public void onFailure(Throwable t) {
-          LOG.warn("NodeEnabledThread exited with error", t);
-        }
-      });
-      schedulerFuture = schedulerExecutor.submit(schedulerCallable);
-      Futures.addCallback(schedulerFuture, new FutureCallback<Void>() {
-        @Override
-        public void onSuccess(Void result) {
-          LOG.info("SchedulerThread exited");
-        }
-
-        @Override
-        public void onFailure(Throwable t) {
-          LOG.warn("SchedulerThread exited with error", t);
-        }
-      });
-      registry.start();
-      activeInstances = registry.getInstances();
-      for (ServiceInstance inst : activeInstances.getAll().values()) {
-        addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public void shutdown() {
-    writeLock.lock();
-    try {
-      if (!this.isStopped.getAndSet(true)) {
-        nodeEnablerCallable.shutdown();
-        if (nodeEnablerFuture != null) {
-          nodeEnablerFuture.cancel(true);
-        }
-        nodeEnabledExecutor.shutdownNow();
-
-        schedulerCallable.shutdown();
-        if (schedulerFuture != null) {
-          schedulerFuture.cancel(true);
-        }
-        schedulerExecutor.shutdownNow();
-
-        if (registry != null) {
-          registry.stop();
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public Resource getTotalResources() {
-    int memory = 0;
-    int vcores = 0;
-    readLock.lock();
-    try {
-      for (ServiceInstance inst : activeInstances.getAll().values()) {
-        if (inst.isAlive()) {
-          Resource r = inst.getResource();
-          LOG.info("Found instance " + inst);
-          memory += r.getMemory();
-          vcores += r.getVirtualCores();
-        } else {
-          LOG.info("Ignoring dead instance " + inst);
-        }
-      }
-    } finally {
-      readLock.unlock();
-    }
-
-    return Resource.newInstance(memory, vcores);
-  }
-
-  /**
-   * The difference between this and getTotalResources() is that this only gives currently free
-   * resource instances, while the other lists all the instances that may become available in a
-   * while.
-   */
-  @Override
-  public Resource getAvailableResources() {
-    // need a state store eventually for current state & measure backoffs
-    int memory = 0;
-    int vcores = 0;
-    readLock.lock();
-    try {
-      for (Entry<ServiceInstance, NodeInfo> entry : instanceToNodeMap.entrySet()) {
-        if (entry.getKey().isAlive() && !entry.getValue().isDisabled()) {
-          Resource r = entry.getKey().getResource();
-          memory += r.getMemory();
-          vcores += r.getVirtualCores();
-        }
-      }
-    } finally {
-      readLock.unlock();
-    }
-
-    return Resource.newInstance(memory, vcores);
-  }
-
-  @Override
-  public int getClusterNodeCount() {
-    readLock.lock();
-    try {
-      int n = 0;
-      for (ServiceInstance inst : activeInstances.getAll().values()) {
-        if (inst.isAlive()) {
-          n++;
-        }
-      }
-      return n;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  public void dagComplete() {
-    // This is effectively DAG completed, and can be used to reset statistics being tracked.
-    LOG.info("DAG: " + dagCounter.get() + " completed. Scheduling stats: " + dagStats);
-    dagCounter.incrementAndGet();
-    dagStats = new StatsPerDag();
-  }
-
-  @Override
-  public void blacklistNode(NodeId nodeId) {
-    LOG.info("BlacklistNode not supported");
-  }
-
-  @Override
-  public void unblacklistNode(NodeId nodeId) {
-    LOG.info("unBlacklistNode not supported");
-  }
-
-  @Override
-  public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
-      Priority priority, Object containerSignature, Object clientCookie) {
-    TaskInfo taskInfo =
-        new TaskInfo(task, clientCookie, priority, capability, hosts, racks, clock.getTime());
-    writeLock.lock();
-    try {
-      dagStats.registerTaskRequest(hosts, racks);
-    } finally {
-      writeLock.unlock();
-    }
-    addPendingTask(taskInfo);
-    trySchedulingPendingTasks();
-  }
-
-  @Override
-  public void allocateTask(Object task, Resource capability, ContainerId containerId,
-      Priority priority, Object containerSignature, Object clientCookie) {
-    // Container affinity can be implemented as Host affinity for LLAP. Not required until
-    // 1:1 edges are used in Hive.
-    TaskInfo taskInfo =
-        new TaskInfo(task, clientCookie, priority, capability, null, null, clock.getTime());
-    writeLock.lock();
-    try {
-      dagStats.registerTaskRequest(null, null);
-    } finally {
-      writeLock.unlock();
-    }
-    addPendingTask(taskInfo);
-    trySchedulingPendingTasks();
-  }
-
-
-  // This may be invoked before a container is ever assigned to a task. allocateTask... app decides
-  // the task is no longer required, and asks for a de-allocation.
-  @Override
-  public boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason, String diagnostics) {
-    writeLock.lock(); // Updating several local structures
-    TaskInfo taskInfo;
-    try {
-      taskInfo = unregisterTask(task);
-      if (taskInfo == null) {
-        LOG.error("Could not determine ContainerId for task: "
-            + task
-            + " . Could have hit a race condition. Ignoring."
-            + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
-        return false;
-      }
-      if (taskInfo.containerId == null) {
-        if (taskInfo.assigned) {
-          LOG.error("Task: "
-              + task
-              + " assigned, but could not find the corresponding containerId."
-              + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
-        } else {
-          LOG.info("Ignoring deallocate request for task " + task
-              + " which hasn't been assigned to a container");
-          removePendingTask(taskInfo);
-        }
-        return false;
-      }
-      ServiceInstance assignedInstance = taskInfo.assignedInstance;
-      assert assignedInstance != null;
-
-      NodeInfo nodeInfo = instanceToNodeMap.get(assignedInstance);
-      assert nodeInfo != null;
-
-      // Re-enable the node if preempted
-      if (taskInfo.preempted) {
-        LOG.info("Processing deallocateTask for {} which was preempted, EndReason={}", task, endReason);
-        unregisterPendingPreemption(taskInfo.assignedInstance.getHost());
-        nodeInfo.registerUnsuccessfulTaskEnd(true);
-        if (nodeInfo.isDisabled()) {
-          // Re-enable the node. If a task succeeded, a slot may have become available.
-          // Also reset commFailures since a task was able to communicate back and indicate success.
-          nodeInfo.enableNode();
-          // Re-insert into the queue to force the poll thread to remove the element.
-          if ( disabledNodesQueue.remove(nodeInfo)) {
-            disabledNodesQueue.add(nodeInfo);
-          }
-        }
-        // In case of success, trigger a scheduling run for pending tasks.
-        trySchedulingPendingTasks();
-      } else {
-        if (taskSucceeded) {
-          // The node may have been blacklisted at this point - which means it may not be in the
-          // activeNodeList.
-
-          nodeInfo.registerTaskSuccess();
-
-          if (nodeInfo.isDisabled()) {
-            // Re-enable the node. If a task succeeded, a slot may have become available.
-            // Also reset commFailures since a task was able to communicate back and indicate success.
-            nodeInfo.enableNode();
-            // Re-insert into the queue to force the poll thread to remove the element.
-            if (disabledNodesQueue.remove(nodeInfo)) {
-              disabledNodesQueue.add(nodeInfo);
-            }
-          }
-          // In case of success, trigger a scheduling run for pending tasks.
-          trySchedulingPendingTasks();
-
-        } else if (!taskSucceeded) {
-          nodeInfo.registerUnsuccessfulTaskEnd(false);
-          if (endReason != null && EnumSet
-              .of(TaskAttemptEndReason.EXECUTOR_BUSY, TaskAttemptEndReason.COMMUNICATION_ERROR)
-              .contains(endReason)) {
-            if (endReason == TaskAttemptEndReason.COMMUNICATION_ERROR) {
-              dagStats.registerCommFailure(taskInfo.assignedInstance.getHost());
-            } else if (endReason == TaskAttemptEndReason.EXECUTOR_BUSY) {
-              dagStats.registerTaskRejected(taskInfo.assignedInstance.getHost());
-            }
-          }
-          boolean commFailure =
-              endReason != null && endReason == TaskAttemptEndReason.COMMUNICATION_ERROR;
-          disableInstance(assignedInstance, commFailure);
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-    getContext().containerBeingReleased(taskInfo.containerId);
-    return true;
-  }
-
-  @Override
-  public Object deallocateContainer(ContainerId containerId) {
-    LOG.debug("Ignoring deallocateContainer for containerId: " + containerId);
-    // Containers are not being tracked for re-use.
-    // This is safe to ignore since a deallocate task will come in.
-    return null;
-  }
-
-  @Override
-  public void setShouldUnregister() {
-
-  }
-
-  @Override
-  public boolean hasUnregistered() {
-    // Nothing to do. No registration involved.
-    return true;
-  }
-
-  private ExecutorService createAppCallbackExecutorService() {
-    return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
-        .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
-  }
-
-  /**
-   * @param request the list of preferred hosts. null implies any host
-   * @return
-   */
-  private SelectHostResult selectHost(TaskInfo request) {
-    String[] requestedHosts = request.requestedHosts;
-    readLock.lock(); // Read-lock. Not updating any stats at the moment.
-    try {
-      // Check if any hosts are active.
-      if (getAvailableResources().getMemory() <= 0) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Refreshing instances since total memory is 0");
-        }
-        refreshInstances();
-      }
-
-      // If there's no memory available, fail
-      if (getTotalResources().getMemory() <= 0) {
-        return SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY;
-      }
-
-      if (requestedHosts != null && requestedHosts.length > 0) {
-        int prefHostCount = -1;
-        boolean requestedHostExists = false;
-        for (String host : requestedHosts) {
-          prefHostCount++;
-          // Pick the first host always. Weak attempt at cache affinity.
-          Set<ServiceInstance> instances = activeInstances.getByHost(host);
-          if (!instances.isEmpty()) {
-            requestedHostExists = true;
-            for (ServiceInstance inst : instances) {
-              NodeInfo nodeInfo = instanceToNodeMap.get(inst);
-              if (nodeInfo != null && nodeInfo.canAcceptTask()) {
-                LOG.info("Assigning " + inst + " when looking for " + host + "." +
-                    " FirstRequestedHost=" + (prefHostCount == 0) +
-                    (requestedHosts.length > 1 ? "#prefLocations=" + requestedHosts.length : ""));
-                return new SelectHostResult(inst, nodeInfo);
-              }
-            }
-          }
-        }
-        // Check if forcing the location is required.
-        if (forceLocation) {
-          if (requestedHostExists) {
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Skipping non-local location allocation for [" + request.task +
-                  "] when trying to allocate on [" + Arrays.toString(requestedHosts) + "]");
-            }
-            return SELECT_HOST_RESULT_DELAYED_LOCALITY;
-          } else {
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Not skipping non-local location allocation for [" + request.task +
-                  "] when trying to allocate on [" + Arrays.toString(requestedHosts) +
-                  "] since none of these hosts are part of the known list");
-            }
-          }
-        }
-      }
-      /* fall through - miss in locality (random scheduling) */
-      Entry<ServiceInstance, NodeInfo>[] all =
-          instanceToNodeMap.entrySet().toArray(new Entry[instanceToNodeMap.size()]);
-      // Check again
-      if (all.length > 0) {
-        int n = random.nextInt(all.length);
-        // start at random offset and iterate whole list
-        for (int i = 0; i < all.length; i++) {
-          Entry<ServiceInstance, NodeInfo> inst = all[(i + n) % all.length];
-          if (inst.getValue().canAcceptTask()) {
-            LOG.info("Assigning " + inst + " when looking for any host, from #hosts=" + all.length +
-                ", requestedHosts=" +
-                ((requestedHosts == null || requestedHosts.length == 0) ? "null" :
-                    Arrays.toString(requestedHosts)));
-            return new SelectHostResult(inst.getKey(), inst.getValue());
-          }
-        }
-      }
-      return SELECT_HOST_RESULT_DELAYED_RESOURCES;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  // TODO Each refresh operation should addNodes if they don't already exist.
-  // Even better would be to get notifications from the service impl when a node gets added or removed.
-  // Instead of having to walk through the entire list. The computation of a node getting added or
-  // removed already exists in the DynamicRegistry implementation.
-  private void refreshInstances() {
-    try {
-      activeInstances.refresh(); // handles its own sync
-    } catch (IOException ioe) {
-      LOG.warn("Could not refresh list of active instances", ioe);
-    }
-  }
-
-  private void scanForNodeChanges() {
-    /* check again whether nodes are disabled or just missing */
-    writeLock.lock();
-    try {
-      for (ServiceInstance inst : activeInstances.getAll().values()) {
-        if (inst.isAlive() && instanceToNodeMap.containsKey(inst) == false) {
-          /* that's a good node, not added to the allocations yet */
-          LOG.info("Found a new node: " + inst + ".");
-          addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void addNode(ServiceInstance inst, NodeInfo node) {
-    LOG.info("Adding node: " + inst);
-    instanceToNodeMap.put(inst, node);
-    // Trigger scheduling since a new node became available.
-    trySchedulingPendingTasks();
-  }
-
-  private void reenableDisabledNode(NodeInfo nodeInfo) {
-    writeLock.lock();
-    try {
-      if (nodeInfo.hadCommFailure()) {
-        // If the node being re-enabled was not marked busy previously, then it was disabled due to
-        // some other failure. Refresh the service list to see if it's been removed permanently.
-        refreshInstances();
-      }
-      LOG.info("Attempting to re-enable node: " + nodeInfo.getServiceInstance());
-      if (nodeInfo.getServiceInstance().isAlive()) {
-        nodeInfo.enableNode();
-      } else {
-        if (LOG.isInfoEnabled()) {
-          LOG.info("Removing dead node " + nodeInfo);
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void disableInstance(ServiceInstance instance, boolean isCommFailure) {
-    writeLock.lock();
-    try {
-      NodeInfo nodeInfo = instanceToNodeMap.get(instance);
-      if (nodeInfo == null || nodeInfo.isDisabled()) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Node: " + instance + " already disabled, or invalid. Not doing anything.");
-        }
-      } else {
-        nodeInfo.disableNode(isCommFailure);
-        // TODO: handle task to container map events in case of hard failures
-        disabledNodesQueue.add(nodeInfo);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void addPendingTask(TaskInfo taskInfo) {
-    writeLock.lock();
-    try {
-      List<TaskInfo> tasksAtPriority = pendingTasks.get(taskInfo.priority);
-      if (tasksAtPriority == null) {
-        tasksAtPriority = new LinkedList<>();
-        pendingTasks.put(taskInfo.priority, tasksAtPriority);
-      }
-      tasksAtPriority.add(taskInfo);
-      knownTasks.putIfAbsent(taskInfo.task, taskInfo);
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  /* Remove a task from the pending list */
-  private void removePendingTask(TaskInfo taskInfo) {
-    writeLock.lock();
-    try {
-      Priority priority = taskInfo.priority;
-      List<TaskInfo> taskInfoList = pendingTasks.get(priority);
-      if (taskInfoList == null || taskInfoList.isEmpty() || !taskInfoList.remove(taskInfo)) {
-        LOG.warn("Could not find task: " + taskInfo.task + " in pending list, at priority: "
-            + priority);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  /* Register a running task into the runningTasks structure */
-  private void registerRunningTask(TaskInfo taskInfo) {
-    writeLock.lock();
-    try {
-      int priority = taskInfo.priority.getPriority();
-      TreeSet<TaskInfo> tasksAtpriority = runningTasks.get(priority);
-      if (tasksAtpriority == null) {
-        tasksAtpriority = new TreeSet<>(TASK_INFO_COMPARATOR);
-        runningTasks.put(priority, tasksAtpriority);
-      }
-      tasksAtpriority.add(taskInfo);
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  /* Unregister a task from the known and running structures */
-  private TaskInfo unregisterTask(Object task) {
-    writeLock.lock();
-    try {
-      TaskInfo taskInfo = knownTasks.remove(task);
-      if (taskInfo != null) {
-        if (taskInfo.assigned) {
-          // Remove from the running list.
-          int priority = taskInfo.priority.getPriority();
-          Set<TaskInfo> tasksAtPriority = runningTasks.get(priority);
-          Preconditions.checkState(tasksAtPriority != null,
-              "runningTasks should contain an entry if the task was in running state. Caused by task: {}", task);
-          tasksAtPriority.remove(taskInfo);
-          if (tasksAtPriority.isEmpty()) {
-            runningTasks.remove(priority);
-          }
-        }
-      } else {
-        LOG.warn("Could not find TaskInfo for task: {}. Not removing it from the running set", task);
-      }
-      return taskInfo;
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private enum ScheduleResult {
-    // Successfully scheduled
-    SCHEDULED,
-
-    // Delayed to find a local match
-    DELAYED_LOCALITY,
-
-    // Delayed due to temporary resource availability
-    DELAYED_RESOURCES,
-
-    // Inadequate total resources - will never succeed / wait for new executors to become available
-    INADEQUATE_TOTAL_RESOURCES,
-  }
-
-  @VisibleForTesting
-  protected void schedulePendingTasks() {
-    writeLock.lock();
-    try {
-      Iterator<Entry<Priority, List<TaskInfo>>> pendingIterator =
-          pendingTasks.entrySet().iterator();
-      while (pendingIterator.hasNext()) {
-        Entry<Priority, List<TaskInfo>> entry = pendingIterator.next();
-        List<TaskInfo> taskListAtPriority = entry.getValue();
-        Iterator<TaskInfo> taskIter = taskListAtPriority.iterator();
-        boolean scheduledAllAtPriority = true;
-        while (taskIter.hasNext()) {
-
-          // TODO Optimization: Add a check to see if there's any capacity available. No point in
-          // walking through all active nodes, if they don't have potential capacity.
-
-          TaskInfo taskInfo = taskIter.next();
-          if (taskInfo.getNumPreviousAssignAttempts() == 1) {
-            dagStats.registerDelayedAllocation();
-          }
-          taskInfo.triedAssigningTask();
-          ScheduleResult scheduleResult = scheduleTask(taskInfo);
-          if (scheduleResult == ScheduleResult.SCHEDULED) {
-            taskIter.remove();
-          } else {
-            // TODO Handle INADEQUATE_TOTAL_RESOURCES eventually - either by throwin an error immediately,
-            // or waiting for some timeout for new executors and then throwing an error
-
-            // Try pre-empting a task so that a higher priority task can take it's place.
-            // Preempt only if there's no pending preemptions to avoid preempting twice for a task.
-            String[] potentialHosts;
-            if (scheduleResult == ScheduleResult.DELAYED_LOCALITY) {
-              // preempt only on specific hosts, if no preemptions already exist on those.
-              potentialHosts = taskInfo.requestedHosts;
-              //Protect against a bad location being requested.
-              if (potentialHosts == null || potentialHosts.length == 0) {
-                potentialHosts = null;
-              }
-            } else {
-              // preempt on any host.
-              potentialHosts = null;
-            }
-
-            if (potentialHosts != null) {
-              // Preempt on specific host
-              boolean shouldPreempt = true;
-              for (String host : potentialHosts) {
-                // Preempt only if there are not pending preemptions on the same host
-                // When the premption registers, the request at the highest priority will be given the slot,
-                // even if the initial request was for some other task.
-                // TODO Maybe register which task the preemption was for, to avoid a bad non-local allocation.
-                MutableInt pendingHostPreemptions = pendingPreemptionsPerHost.get(host);
-                if (pendingHostPreemptions != null && pendingHostPreemptions.intValue() > 0) {
-                  shouldPreempt = false;
-                  break;
-                }
-              }
-              if (shouldPreempt) {
-                LOG.info("Attempting to preempt for {}, pendingPreemptions={} on hosts={}",
-                    taskInfo.task, pendingPreemptions.get(), Arrays.toString(potentialHosts));
-                preemptTasks(entry.getKey().getPriority(), 1, potentialHosts);
-              }
-            } else {
-              // Request for a preemption if there's none pending. If a single preemption is pending,
-              // and this is the next task to be assigned, it will be assigned once that slot becomes available.
-              if (pendingPreemptions.get() == 0) {
-                LOG.info("Attempting to preempt for {}, pendingPreemptions={} on any host",
-                    taskInfo.task, pendingPreemptions.get());
-                preemptTasks(entry.getKey().getPriority(), 1, null);
-              }
-            }
-            // Since there was an allocation failure - don't try assigning tasks at the next priority.
-
-            scheduledAllAtPriority = false;
-            // Don't break if this allocation failure was a result of a LOCALITY_DELAY. Others could still be allocated.
-            if (scheduleResult != ScheduleResult.DELAYED_LOCALITY) {
-              break;
-            }
-          } // end of else - i.e. could not allocate
-        } // end of loop over pending tasks
-        if (taskListAtPriority.isEmpty()) {
-          // Remove the entry, if there's nothing left at the specific priority level
-          pendingIterator.remove();
-        }
-        if (!scheduledAllAtPriority) {
-          // Don't attempt scheduling for additional priorities
-          break;
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-
-  private ScheduleResult scheduleTask(TaskInfo taskInfo) {
-    SelectHostResult selectHostResult = selectHost(taskInfo);
-    if (selectHostResult.scheduleResult == ScheduleResult.SCHEDULED) {
-      NodeServiceInstancePair nsPair = selectHostResult.nodeServiceInstancePair;
-      Container container =
-          containerFactory.createContainer(resourcePerExecutor, taskInfo.priority,
-              nsPair.getServiceInstance().getHost(),
-              nsPair.getServiceInstance().getRpcPort());
-      writeLock.lock(); // While updating local structures
-      try {
-        LOG.info("Assigned task {} to container {}", taskInfo, container.getId());
-        dagStats.registerTaskAllocated(taskInfo.requestedHosts, taskInfo.requestedRacks,
-            nsPair.getServiceInstance().getHost());
-        taskInfo.setAssignmentInfo(nsPair.getServiceInstance(), container.getId(), clock.getTime());
-        registerRunningTask(taskInfo);
-        nsPair.getNodeInfo().registerTaskScheduled();
-      } finally {
-        writeLock.unlock();
-      }
-      getContext().taskAllocated(taskInfo.task, taskInfo.clientCookie, container);
-    }
-    return selectHostResult.scheduleResult;
-  }
-
-  // Removes tasks from the runningList and sends out a preempt request to the system.
-  // Subsequent tasks will be scheduled again once the de-allocate request for the preempted
-  // task is processed.
-  private void preemptTasks(int forPriority, int numTasksToPreempt, String []potentialHosts) {
-    Set<String> preemptHosts;
-    if (potentialHosts == null) {
-      preemptHosts = null;
-    } else {
-      preemptHosts = Sets.newHashSet(potentialHosts);
-    }
-    writeLock.lock();
-    List<TaskInfo> preemptedTaskList = null;
-    try {
-      NavigableMap<Integer, TreeSet<TaskInfo>> orderedMap = runningTasks.descendingMap();
-      Iterator<Entry<Integer, TreeSet<TaskInfo>>> iterator = orderedMap.entrySet().iterator();
-      int preemptedCount = 0;
-      while (iterator.hasNext() && preemptedCount < numTasksToPreempt) {
-        Entry<Integer, TreeSet<TaskInfo>> entryAtPriority = iterator.next();
-        if (entryAtPriority.getKey() > forPriority) {
-          Iterator<TaskInfo> taskInfoIterator = entryAtPriority.getValue().iterator();
-          while (taskInfoIterator.hasNext() && preemptedCount < numTasksToPreempt) {
-            TaskInfo taskInfo = taskInfoIterator.next();
-            if (preemptHosts == null || preemptHosts.contains(taskInfo.assignedInstance.getHost())) {
-              // Candidate for preemption.
-              preemptedCount++;
-              LOG.info("preempting {} for task at priority {} with potentialHosts={}", taskInfo,
-                  forPriority, potentialHosts == null ? "" : Arrays.toString(potentialHosts));
-              taskInfo.setPreemptedInfo(clock.getTime());
-              if (preemptedTaskList == null) {
-                preemptedTaskList = new LinkedList<>();
-              }
-              dagStats.registerTaskPreempted(taskInfo.assignedInstance.getHost());
-              preemptedTaskList.add(taskInfo);
-              registerPendingPreemption(taskInfo.assignedInstance.getHost());
-              // Remove from the runningTaskList
-              taskInfoIterator.remove();
-            }
-          }
-
-          // Remove entire priority level if it's been emptied.
-          if (entryAtPriority.getValue().isEmpty()) {
-            iterator.remove();
-          }
-        } else {
-          // No tasks qualify as preemptable
-          LOG.info("DBG: No tasks qualify as killable to schedule tasks at priority {}", forPriority);
-          break;
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-    // Send out the preempted request outside of the lock.
-    if (preemptedTaskList != null) {
-      for (TaskInfo taskInfo : preemptedTaskList) {
-        LOG.info("DBG: Preempting task {}", taskInfo);
-        getContext().preemptContainer(taskInfo.containerId);
-        // Preemption will finally be registered as a deallocateTask as a result of preemptContainer
-        // That resets preemption info and allows additional tasks to be pre-empted if required.
-      }
-    }
-    // The schedule loop will be triggered again when the deallocateTask request comes in for the
-    // preempted task.
-  }
-
-  private void registerPendingPreemption(String host) {
-    writeLock.lock();
-    try {
-      pendingPreemptions.incrementAndGet();
-      MutableInt val = pendingPreemptionsPerHost.get(host);
-      if (val == null) {
-        val = new MutableInt(1);
-        pendingPreemptionsPerHost.put(host, val);
-      }
-      val.increment();
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void unregisterPendingPreemption(String host) {
-    writeLock.lock();
-    try {
-      pendingPreemptions.decrementAndGet();
-      MutableInt val = pendingPreemptionsPerHost.get(host);
-      Preconditions.checkNotNull(val);
-      val.decrement();
-      // Not bothering with removing the entry. There's a limited number of hosts, and a good
-      // chance that the entry will make it back in when the AM is used for a long duration.
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private class NodeEnablerCallable implements Callable<Void> {
-
-    private AtomicBoolean isShutdown = new AtomicBoolean(false);
-    private static final long REFRESH_INTERVAL = 10000l;
-    long nextPollInterval = REFRESH_INTERVAL;
-    long lastRefreshTime;
-
-    @Override
-    public Void call() {
-
-      lastRefreshTime = System.currentTimeMillis();
-      while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
-        try {
-          while (true) {
-            NodeInfo nodeInfo = disabledNodesQueue.poll(nextPollInterval, TimeUnit.MILLISECONDS);
-            if (nodeInfo != null) {
-              long currentTime = System.currentTimeMillis();
-              // A node became available. Enable the node and try scheduling.
-              reenableDisabledNode(nodeInfo);
-              trySchedulingPendingTasks();
-
-              nextPollInterval -= (currentTime - lastRefreshTime);
-            }
-
-            if (nextPollInterval < 0 || nodeInfo == null) {
-              // timeout expired. Reset the poll interval and refresh nodes.
-              nextPollInterval = REFRESH_INTERVAL;
-              lastRefreshTime = System.currentTimeMillis();
-              // TODO Get rid of this polling once we have notificaitons from the registry sub-system
-              if (LOG.isDebugEnabled()) {
-                LOG.debug("Refreshing instances based on poll interval");
-              }
-              refreshInstances();
-              scanForNodeChanges();
-            }
-          }
-        } catch (InterruptedException e) {
-          if (isShutdown.get()) {
-            LOG.info("NodeEnabler thread interrupted after shutdown");
-            break;
-          } else {
-            LOG.warn("NodeEnabler thread interrupted without being shutdown");
-            throw new RuntimeException("NodeEnabler thread interrupted without being shutdown", e);
-          }
-        }
-      }
-      return null;
-    }
-
-    // Call this first, then send in an interrupt to the thread.
-    public void shutdown() {
-      isShutdown.set(true);
-    }
-  }
-
-  private void trySchedulingPendingTasks() {
-    scheduleLock.lock();
-    try {
-      pendingScheduleInvodations.set(true);
-      scheduleCondition.signal();
-    } finally {
-      scheduleLock.unlock();
-    }
-  }
-
-  private class SchedulerCallable implements Callable<Void> {
-    private AtomicBoolean isShutdown = new AtomicBoolean(false);
-
-    @Override
-    public Void call() {
-      while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
-        scheduleLock.lock();
-        try {
-          while (!pendingScheduleInvodations.get()) {
-            scheduleCondition.await();
-          }
-        } catch (InterruptedException e) {
-          if (isShutdown.get()) {
-            LOG.info("Scheduler thread interrupted after shutdown");
-            break;
-          } else {
-            LOG.warn("Scheduler thread interrupted without being shutdown");
-            throw new RuntimeException("Scheduler thread interrupted without being shutdown", e);
-          }
-        } finally {
-          scheduleLock.unlock();
-        }
-
-        // Set pending to false since scheduling is about to run. Any triggers up to this point
-        // will be handled in the next run.
-        // A new request may come in right after this is set to false, but before the actual scheduling.
-        // This will be handled in this run, but will cause an immediate run after, which is harmless.
-        pendingScheduleInvodations.set(false);
-        // Schedule outside of the scheduleLock - which should only be used to wait on the condition.
-        schedulePendingTasks();
-      }
-      return null;
-    }
-
-    // Call this first, then send in an interrupt to the thread.
-    public void shutdown() {
-      isShutdown.set(true);
-    }
-  }
-
-  @VisibleForTesting
-  static class NodeInfo implements Delayed {
-    private final NodeBlacklistConf blacklistConf;
-    final ServiceInstance serviceInstance;
-    private final Clock clock;
-
-    long expireTimeMillis = -1;
-    private long numSuccessfulTasks = 0;
-    private long numSuccessfulTasksAtLastBlacklist = -1;
-    float cumulativeBackoffFactor = 1.0f;
-
-    // Indicates whether a node had a recent communication failure.
-    private boolean hadCommFailure = false;
-
-    // Indicates whether a node is disabled - for whatever reason - commFailure, busy, etc.
-    private boolean disabled = false;
-
-    private int numPreemptedTasks = 0;
-    private int numScheduledTasks = 0;
-    private final int numSchedulableTasks;
-
-
-    /**
-     * Create a NodeInfo bound to a service instance
-     *
-     * @param serviceInstance         the associated serviceInstance
-     * @param blacklistConf           blacklist configuration
-     * @param clock                   clock to use to obtain timing information
-     * @param numSchedulableTasksConf number of schedulable tasks on the node. 0 represents auto
-     *                                detect based on the serviceInstance, -1 indicates indicates
-     *                                unlimited capacity
-     */
-    NodeInfo(ServiceInstance serviceInstance, NodeBlacklistConf blacklistConf, Clock clock, int numSchedulableTasksConf) {
-      Preconditions.checkArgument(numSchedulableTasksConf >= -1, "NumSchedulableTasks must be >=-1");
-      this.serviceInstance = serviceInstance;
-      this.blacklistConf = blacklistConf;
-      this.clock = clock;
-
-      if (numSchedulableTasksConf == 0) {
-        int pendingQueueuCapacity = 0;
-        String pendingQueueCapacityString = serviceInstance.getProperties()
-            .get(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname);
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Setting up node: " + serviceInstance + ", with available capacity=" +
-              serviceInstance.getResource().getVirtualCores() + ", pendingQueueCapacity=" +
-              pendingQueueCapacityString);
-        }
-
-        if (pendingQueueCapacityString != null) {
-          pendingQueueuCapacity = Integer.parseInt(pendingQueueCapacityString);
-        }
-        this.numSchedulableTasks = serviceInstance.getResource().getVirtualCores() + pendingQueueuCapacity;
-      } else {
-        this.numSchedulableTasks = numSchedulableTasksConf;
-      }
-      LOG.info("Setting up node: " + serviceInstance + " with schedulableCapacity=" + this.numSchedulableTasks);
-    }
-
-    ServiceInstance getServiceInstance() {
-      return serviceInstance;
-    }
-
-    void enableNode() {
-      expireTimeMillis = -1;
-      disabled = false;
-      hadCommFailure = false;
-    }
-
-    void disableNode(boolean commFailure) {
-      long duration = blacklistConf.minDelay;
-      long currentTime = clock.getTime();
-      this.hadCommFailure = commFailure;
-      disabled = true;
-      if (numSuccessfulTasksAtLastBlacklist == numSuccessfulTasks) {
-        // Relying on a task succeeding to reset the exponent.
-        // There's no notifications on whether a task gets accepted or not. That would be ideal to
-        // reset this.
-        cumulativeBackoffFactor = cumulativeBackoffFactor * blacklistConf.backoffFactor;
-      } else {
-        // Was able to execute something before the last blacklist. Reset the exponent.
-        cumulativeBackoffFactor = 1.0f;
-      }
-
-      long delayTime = (long) (duration * cumulativeBackoffFactor);
-      if (delayTime > blacklistConf.maxDelay) {
-        delayTime = blacklistConf.maxDelay;
-      }
-      if (LOG.isInfoEnabled()) {
-        LOG.info("Disabling instance " + serviceInstance + " for " + delayTime + " milli-seconds");
-      }
-      expireTimeMillis = currentTime + delayTime;
-      numSuccessfulTasksAtLastBlacklist = numSuccessfulTasks;
-    }
-
-    void registerTaskScheduled() {
-      numScheduledTasks++;
-    }
-
-    void registerTaskSuccess() {
-      numSuccessfulTasks++;
-      numScheduledTasks--;
-    }
-
-    void registerUnsuccessfulTaskEnd(boolean wasPreempted) {
-      numScheduledTasks--;
-      if (wasPreempted) {
-        numPreemptedTasks++;
-      }
-    }
-
-    public boolean isDisabled() {
-      return disabled;
-    }
-
-    public boolean hadCommFailure() {
-      return hadCommFailure;
-    }
-    /* Returning true does not guarantee that the task will run, considering other queries
-    may be running in the system. Also depends upon the capacity usage configuration
-     */
-    public boolean canAcceptTask() {
-      boolean result = !hadCommFailure && !disabled && serviceInstance.isAlive()
-          &&(numSchedulableTasks == -1 || ((numSchedulableTasks - numScheduledTasks) > 0));
-      LOG.info("canAcceptTask={}, numScheduledTasks={}, numSchedulableTasks={}, hadCommFailure={}, disabled={}, serviceInstance.isAlive={}", result, numScheduledTasks, numSchedulableTasks, hadCommFailure, disabled, serviceInstance.isAlive());
-      return result;
-    }
-
-    @Override
-    public long getDelay(TimeUnit unit) {
-      return unit.convert(expireTimeMillis - clock.getTime(), TimeUnit.MILLISECONDS);
-    }
-
-    @Override
-    public int compareTo(Delayed o) {
-      NodeInfo other = (NodeInfo) o;
-      if (other.expireTimeMillis > this.expireTimeMillis) {
-        return -1;
-      } else if (other.expireTimeMillis < this.expireTimeMillis) {
-        return 1;
-      } else {
-        return 0;
-      }
-    }
-
-    @Override
-    public String toString() {
-      return "NodeInfo{" + "instance=" + serviceInstance
-          + ", expireTimeMillis=" + expireTimeMillis + ", numSuccessfulTasks=" + numSuccessfulTasks
-          + ", numSuccessfulTasksAtLastBlacklist=" + numSuccessfulTasksAtLastBlacklist
-          + ", cumulativeBackoffFactor=" + cumulativeBackoffFactor
-          + ", numSchedulableTasks=" + numSchedulableTasks
-          + ", numScheduledTasks=" + numScheduledTasks
-          + ", disabled=" + disabled
-          + ", commFailures=" + hadCommFailure
-          +'}';
-    }
-  }
-
-  @VisibleForTesting
-  static class StatsPerDag {
-    int numRequestedAllocations = 0;
-    int numRequestsWithLocation = 0;
-    int numRequestsWithoutLocation = 0;
-    int numTotalAllocations = 0;
-    int numLocalAllocations = 0;
-    int numNonLocalAllocations = 0;
-    int numAllocationsNoLocalityRequest = 0;
-    int numRejectedTasks = 0;
-    int numCommFailures = 0;
-    int numDelayedAllocations = 0;
-    int numPreemptedTasks = 0;
-    Map<String, AtomicInteger> localityBasedNumAllocationsPerHost = new HashMap<>();
-    Map<String, AtomicInteger> numAllocationsPerHost = new HashMap<>();
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder();
-      sb.append("NumPreemptedTasks=").append(numPreemptedTasks).append(", ");
-      sb.append("NumRequestedAllocations=").append(numRequestedAllocations).append(", ");
-      sb.append("NumRequestsWithlocation=").append(numRequestsWithLocation).append(", ");
-      sb.append("NumLocalAllocations=").append(numLocalAllocations).append(",");
-      sb.append("NumNonLocalAllocations=").append(numNonLocalAllocations).append(",");
-      sb.append("NumTotalAllocations=").append(numTotalAllocations).append(",");
-      sb.append("NumRequestsWithoutLocation=").append(numRequestsWithoutLocation).append(", ");
-      sb.append("NumRejectedTasks=").append(numRejectedTasks).append(", ");
-      sb.append("NumCommFailures=").append(numCommFailures).append(", ");
-      sb.append("NumDelayedAllocations=").append(numDelayedAllocations).append(", ");
-      sb.append("LocalityBasedAllocationsPerHost=").append(localityBasedNumAllocationsPerHost)
-          .append(", ");
-      sb.append("NumAllocationsPerHost=").append(numAllocationsPerHost);
-      return sb.toString();
-    }
-
-    void registerTaskRequest(String[] requestedHosts, String[] requestedRacks) {
-      numRequestedAllocations++;
-      // TODO Change after HIVE-9987. For now, there's no rack matching.
-      if (requestedHosts != null && requestedHosts.length != 0) {
-        numRequestsWithLocation++;
-      } else {
-        numRequestsWithoutLocation++;
-      }
-    }
-
-    void registerTaskAllocated(String[] requestedHosts, String[] requestedRacks,
-        String allocatedHost) {
-      // TODO Change after HIVE-9987. For now, there's no rack matching.
-      if (requestedHosts != null && requestedHosts.length != 0) {
-        Set<String> requestedHostSet = new HashSet<>(Arrays.asList(requestedHosts));
-        if (requestedHostSet.contains(allocatedHost)) {
-          numLocalAllocations++;
-          _registerAllocationInHostMap(allocatedHost, localityBasedNumAllocationsPerHost);
-        } else {
-          numNonLocalAllocations++;
-        }
-      } else {
-        numAllocationsNoLocalityRequest++;
-      }
-      numTotalAllocations++;
-      _registerAllocationInHostMap(allocatedHost, numAllocationsPerHost);
-    }
-
-    void registerTaskPreempted(String host) {
-      numPreemptedTasks++;
-    }
-
-    void registerCommFailure(String host) {
-      numCommFailures++;
-    }
-
-    void registerTaskRejected(String host) {
-      numRejectedTasks++;
-    }
-
-    void registerDelayedAllocation() {
-      numDelayedAllocations++;
-    }
-
-    private void _registerAllocationInHostMap(String host, Map<String, AtomicInteger> hostMap) {
-      AtomicInteger val = hostMap.get(host);
-      if (val == null) {
-        val = new AtomicInteger(0);
-        hostMap.put(host, val);
-      }
-      val.incrementAndGet();
-    }
-  }
-
-  private static class TaskInfo {
-    // IDs used to ensure two TaskInfos are different without using the underlying task instance.
-    // Required for insertion into a TreeMap
-    static final AtomicLong ID_GEN = new AtomicLong(0);
-    final long uniqueId;
-    final Object task;
-    final Object clientCookie;
-    final Priority priority;
-    final Resource capability;
-    final String[] requestedHosts;
-    final String[] requestedRacks;
-    final long requestTime;
-    long startTime;
-    long preemptTime;
-    ContainerId containerId;
-    ServiceInstance assignedInstance;
-    private boolean assigned = false;
-    private boolean preempted = false;
-
-    private int numAssignAttempts = 0;
-
-    // TaskInfo instances for two different tasks will not be the same. Only a single instance should
-    // ever be created for a taskAttempt
-    public TaskInfo(Object task, Object clientCookie, Priority priority, Resource capability,
-        String[] hosts, String[] racks, long requestTime) {
-      this.task = task;
-      this.clientCookie = clientCookie;
-      this.priority = priority;
-      this.capability = capability;
-      this.requestedHosts = hosts;
-      this.requestedRacks = racks;
-      this.requestTime = requestTime;
-      this.uniqueId = ID_GEN.getAndIncrement();
-    }
-
-    void setAssignmentInfo(ServiceInstance instance, ContainerId containerId, long startTime) {
-      this.assignedInstance = instance;
-        this.containerId = containerId;
-      this.startTime = startTime;
-      assigned = true;
-    }
-
-    void setPreemptedInfo(long preemptTime) {
-      this.preempted = true;
-      this.assigned = false;
-      this.preemptTime = preemptTime;
-    }
-
-    void triedAssigningTask() {
-      numAssignAttempts++;
-    }
-
-    int getNumPreviousAssignAttempts() {
-      return numAssignAttempts;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-      if (this == o) {
-        return true;
-      }
-      if (o == null || getClass() != o.getClass()) {
-        return false;
-      }
-
-      TaskInfo taskInfo = (TaskInfo) o;
-
-      if (uniqueId != taskInfo.uniqueId) {
-        return false;
-      }
-      return task.equals(taskInfo.task);
-
-    }
-
-    @Override
-    public int hashCode() {
-      int result = (int) (uniqueId ^ (uniqueId >>> 32));
-      result = 31 * result + task.hashCode();
-      return result;
-    }
-
-    @Override
-    public String toString() {
-      return "TaskInfo{" +
-          "task=" + task +
-          ", priority=" + priority +
-          ", startTime=" + startTime +
-          ", containerId=" + containerId +
-          ", assignedInstance=" + assignedInstance +
-          ", uniqueId=" + uniqueId +
-          '}';
-    }
-  }
-
-  // Newer tasks first.
-  private static class TaskStartComparator implements Comparator<TaskInfo> {
-
-    @Override
-    public int compare(TaskInfo o1, TaskInfo o2) {
-      if (o1.startTime > o2.startTime) {
-        return -1;
-      } else if (o1.startTime < o2.startTime) {
-        return 1;
-      } else {
-        // Comparing on time is not sufficient since two may be created at the same time,
-        // in which case inserting into a TreeSet/Map would break
-        if (o1.uniqueId > o2.uniqueId) {
-          return -1;
-        } else if (o1.uniqueId < o2.uniqueId) {
-          return 1;
-        } else {
-          return 0;
-        }
-      }
-    }
-  }
-
-  private static class SelectHostResult {
-    final NodeServiceInstancePair nodeServiceInstancePair;
-    final ScheduleResult scheduleResult;
-
-    SelectHostResult(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
-      this.nodeServiceInstancePair = new NodeServiceInstancePair(serviceInstance, nodeInfo);
-      this.scheduleResult = ScheduleResult.SCHEDULED;
-    }
-
-    SelectHostResult(ScheduleResult scheduleResult) {
-      this.nodeServiceInstancePair = null;
-      this.scheduleResult = scheduleResult;
-    }
-  }
-
-  private static final SelectHostResult SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY =
-      new SelectHostResult(ScheduleResult.INADEQUATE_TOTAL_RESOURCES);
-  private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_LOCALITY =
-      new SelectHostResult(ScheduleResult.DELAYED_LOCALITY);
-  private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_RESOURCES =
-      new SelectHostResult(ScheduleResult.DELAYED_RESOURCES);
-
-  private static class NodeServiceInstancePair {
-    final NodeInfo nodeInfo;
-    final ServiceInstance serviceInstance;
-
-    private NodeServiceInstancePair(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
-      this.serviceInstance = serviceInstance;
-      this.nodeInfo = nodeInfo;
-    }
-
-    public ServiceInstance getServiceInstance() {
-      return serviceInstance;
-    }
-
-    public NodeInfo getNodeInfo() {
-      return nodeInfo;
-    }
-  }
-
-  private static final class NodeBlacklistConf {
-    private final long minDelay;
-    private final long maxDelay;
-    private final float backoffFactor;
-
-    public NodeBlacklistConf(long minDelay, long maxDelay, float backoffFactor) {
-      this.minDelay = minDelay;
-      this.maxDelay = maxDelay;
-      this.backoffFactor = backoffFactor;
-    }
-
-    @Override
-    public String toString() {
-      return "NodeBlacklistConf{" +
-          "minDelay=" + minDelay +
-          ", maxDelay=" + maxDelay +
-          ", backoffFactor=" + backoffFactor +
-          '}';
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/protobuf/LlapDaemonProtocol.proto
----------------------------------------------------------------------
diff --git a/llap-server/src/protobuf/LlapDaemonProtocol.proto b/llap-server/src/protobuf/LlapDaemonProtocol.proto
deleted file mode 100644
index 944c96c..0000000
--- a/llap-server/src/protobuf/LlapDaemonProtocol.proto
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.hive.llap.daemon.rpc";
-option java_outer_classname = "LlapDaemonProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-
-// TODO Change this as the interface evolves. Currently using Tez constructs.
-
-message UserPayloadProto {
-  optional bytes user_payload = 1;
-  optional int32 version = 2;
-}
-
-message EntityDescriptorProto {
-  optional string class_name = 1;
-  optional UserPayloadProto user_payload = 2;
-  optional bytes history_text = 3;
-}
-
-message IOSpecProto {
-  optional string connected_vertex_name = 1;
-  optional EntityDescriptorProto io_descriptor = 2;
-  optional int32 physical_edge_count = 3;
-}
-
-message GroupInputSpecProto {
-  optional string group_name = 1;
-  repeated string group_vertices = 2;
-  optional EntityDescriptorProto merged_input_descriptor = 3;
-}
-
-
-message FragmentSpecProto {
-  optional string fragment_identifier_string = 1;
-  optional string dag_name = 2;
-  optional int32 dag_id = 11;
-  optional string vertex_name = 3;
-  optional EntityDescriptorProto processor_descriptor = 4;
-  repeated IOSpecProto input_specs = 5;
-  repeated IOSpecProto output_specs = 6;
-  repeated GroupInputSpecProto grouped_input_specs = 7;
-  optional int32 vertex_parallelism = 8;
-  optional int32 fragment_number =9;
-  optional int32 attempt_number = 10;
-}
-
-message FragmentRuntimeInfo {
-  optional int32 num_self_and_upstream_tasks = 1;
-  optional int32 num_self_and_upstream_completed_tasks = 2;
-  optional int32 within_dag_priority = 3;
-  optional int64 dag_start_time = 4;
-  optional int64 first_attempt_start_time = 5;
-  optional int64 current_attempt_start_time = 6;
-}
-
-enum SourceStateProto {
-  S_SUCCEEDED = 1;
-  S_RUNNING = 2;
-}
-
-message QueryIdentifierProto {
-  optional string app_identifier = 1;
-  optional int32 dag_identifier = 2;
-}
-
-message SubmitWorkRequestProto {
-  optional string container_id_string = 1;
-  optional string am_host = 2;
-  optional int32 am_port = 3;
-  optional string token_identifier = 4;
-  optional bytes credentials_binary = 5;
-  optional string user = 6;
-  optional string application_id_string = 7;
-  optional int32 app_attempt_number = 8;
-  optional FragmentSpecProto fragment_spec = 9;
-  optional FragmentRuntimeInfo fragment_runtime_info = 10;
-}
-
-enum SubmissionStateProto {
-  ACCEPTED = 1;
-  REJECTED = 2;
-  EVICTED_OTHER = 3;
-}
-
-message SubmitWorkResponseProto {
-  optional SubmissionStateProto submission_state = 1;
-}
-
-message SourceStateUpdatedRequestProto {
-  optional QueryIdentifierProto query_identifier = 1;
-  optional string src_name = 2;
-  optional SourceStateProto state = 3;
-}
-
-message SourceStateUpdatedResponseProto {
-}
-
-message QueryCompleteRequestProto {
-  optional string query_id = 1;
-  optional QueryIdentifierProto query_identifier = 2;
-  optional int64 delete_delay = 4 [default = 0];
-}
-
-message QueryCompleteResponseProto {
-}
-
-message TerminateFragmentRequestProto {
-  optional QueryIdentifierProto query_identifier = 1;
-  optional string fragment_identifier_string = 2;
-}
-
-message TerminateFragmentResponseProto {
-}
-
-message GetTokenRequestProto {
-}
-
-message GetTokenResponseProto {
-  optional bytes token = 1;
-}
-
-service LlapDaemonProtocol {
-  rpc submitWork(SubmitWorkRequestProto) returns (SubmitWorkResponseProto);
-  rpc sourceStateUpdated(SourceStateUpdatedRequestProto) returns (SourceStateUpdatedResponseProto);
-  rpc queryComplete(QueryCompleteRequestProto) returns (QueryCompleteResponseProto);
-  rpc terminateFragment(TerminateFragmentRequestProto) returns (TerminateFragmentResponseProto);
-}
-
-service LlapManagementProtocol {
-  rpc getDelegationToken(GetTokenRequestProto) returns (GetTokenResponseProto);
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
index 44c958d..1cef218 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
@@ -27,10 +27,11 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
 import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
 import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmissionStateProto;
+import org.apache.hadoop.hive.llap.impl.LlapProtocolClientImpl;
 import org.junit.Test;
 
 public class TestLlapDaemonProtocolServerImpl {
@@ -42,8 +43,8 @@ public class TestLlapDaemonProtocolServerImpl {
     int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT);
     int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
     ContainerRunner containerRunnerMock = mock(ContainerRunner.class);
-    LlapDaemonProtocolServerImpl server =
-        new LlapDaemonProtocolServerImpl(numHandlers, containerRunnerMock,
+    LlapProtocolServerImpl server =
+        new LlapProtocolServerImpl(numHandlers, containerRunnerMock,
            new AtomicReference<InetSocketAddress>(), new AtomicReference<InetSocketAddress>(),
            rpcPort, rpcPort + 1);
     when(containerRunnerMock.submitWork(any(SubmitWorkRequestProto.class))).thenReturn(
@@ -56,8 +57,8 @@ public class TestLlapDaemonProtocolServerImpl {
       server.start();
       InetSocketAddress serverAddr = server.getBindAddress();
 
-      LlapDaemonProtocolBlockingPB client =
-          new LlapDaemonProtocolClientImpl(new Configuration(), serverAddr.getHostName(),
+      LlapProtocolBlockingPB client =
+          new LlapProtocolClientImpl(new Configuration(), serverAddr.getHostName(),
               serverAddr.getPort(), null, null);
       SubmitWorkResponseProto responseProto = client.submitWork(null,
           SubmitWorkRequestProto.newBuilder()

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java
deleted file mode 100644
index a6af8c2..0000000
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.protobuf.Message;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.junit.Test;
-
-public class TestLlapDaemonProtocolClientProxy {
-
-  @Test (timeout = 5000)
-  public void testMultipleNodes() {
-    RequestManagerForTest requestManager = new RequestManagerForTest(1);
-
-    LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
-    LlapNodeId nodeId2 = LlapNodeId.getInstance("host2", 1025);
-
-    Message mockMessage = mock(Message.class);
-    LlapDaemonProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
-        LlapDaemonProtocolClientProxy.ExecuteRequestCallback.class);
-
-    // Request two messages
-    requestManager.queueRequest(
-        new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
-    requestManager.queueRequest(
-        new CallableRequestForTest(nodeId2, mockMessage, mockExecuteRequestCallback));
-
-    // Should go through in a single process call
-    requestManager.process();
-    assertEquals(2, requestManager.numSubmissionsCounters);
-    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
-    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId2));
-    assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
-    assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId2).getValue().intValue());
-    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
-    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
-    assertEquals(0, requestManager.currentLoopDisabledNodes.size());
-  }
-
-  @Test(timeout = 5000)
-  public void testSingleInvocationPerNode() {
-    RequestManagerForTest requestManager = new RequestManagerForTest(1);
-
-    LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
-
-    Message mockMessage = mock(Message.class);
-    LlapDaemonProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
-        LlapDaemonProtocolClientProxy.ExecuteRequestCallback.class);
-
-    // First request for host.
-    requestManager.queueRequest(
-        new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
-    requestManager.process();
-    assertEquals(1, requestManager.numSubmissionsCounters);
-    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
-    assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
-    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
-
-    // Second request for host. Single invocation since the last has not completed.
-    requestManager.queueRequest(
-        new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
-    requestManager.process();
-    assertEquals(1, requestManager.numSubmissionsCounters);
-    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
-    assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
-    assertEquals(1, requestManager.currentLoopSkippedRequests.size());
-    assertEquals(1, requestManager.currentLoopDisabledNodes.size());
-    assertTrue(requestManager.currentLoopDisabledNodes.contains(nodeId1));
-
-    // Complete first request. Second pending request should go through.
-    requestManager.requestFinished(nodeId1);
-    requestManager.process();
-    assertEquals(2, requestManager.numSubmissionsCounters);
-    assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
-    assertEquals(2, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
-    assertEquals(0, requestManager.currentLoopSkippedRequests.size());
-    assertEquals(0, requestManager.currentLoopDisabledNodes.size());
-    assertFalse(requestManager.currentLoopDisabledNodes.contains(nodeId1));
-  }
-
-
-  static class RequestManagerForTest extends LlapDaemonProtocolClientProxy.RequestManager {
-
-    int numSubmissionsCounters = 0;
-    private Map<LlapNodeId, MutableInt> numInvocationsPerNode = new HashMap<>();
-
-    public RequestManagerForTest(int numThreads) {
-      super(numThreads);
-    }
-
-    protected void submitToExecutor(LlapDaemonProtocolClientProxy.CallableRequest request, LlapNodeId nodeId) {
-      numSubmissionsCounters++;
-      MutableInt nodeCount = numInvocationsPerNode.get(nodeId);
-      if (nodeCount == null) {
-        nodeCount = new MutableInt(0);
-        numInvocationsPerNode.put(nodeId, nodeCount);
-      }
-      nodeCount.increment();
-    }
-
-    void reset() {
-      numSubmissionsCounters = 0;
-      numInvocationsPerNode.clear();
-    }
-
-  }
-
-  static class CallableRequestForTest extends LlapDaemonProtocolClientProxy.CallableRequest<Message, Message> {
-
-    protected CallableRequestForTest(LlapNodeId nodeId, Message message,
-                                     LlapDaemonProtocolClientProxy.ExecuteRequestCallback<Message> callback) {
-      super(nodeId, message, callback);
-    }
-
-    @Override
-    public Message call() throws Exception {
-      return null;
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
deleted file mode 100644
index 8f3d104..0000000
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.junit.Test;
-
-public class TestLlapTaskCommunicator {
-
-  @Test (timeout = 5000)
-  public void testEntityTracker1() {
-    LlapTaskCommunicator.EntityTracker entityTracker = new LlapTaskCommunicator.EntityTracker();
-
-    String host1 = "host1";
-    String host2 = "host2";
-    String host3 = "host3";
-    int port = 1451;
-
-
-    // Simple container registration and un-registration without any task attempt being involved.
-    ContainerId containerId101 = constructContainerId(101);
-    entityTracker.registerContainer(containerId101, host1, port);
-    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId101));
-
-    entityTracker.unregisterContainer(containerId101);
-    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
-    assertNull(entityTracker.getNodeIdForContainer(containerId101));
-    assertEquals(0, entityTracker.nodeMap.size());
-    assertEquals(0, entityTracker.attemptToNodeMap.size());
-    assertEquals(0, entityTracker.containerToNodeMap.size());
-
-
-    // Simple task registration and un-registration.
-    ContainerId containerId1 = constructContainerId(1);
-    TezTaskAttemptID taskAttemptId1 = constructTaskAttemptId(1);
-    entityTracker.registerTaskAttempt(containerId1, taskAttemptId1, host1, port);
-    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId1));
-    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
-
-    entityTracker.unregisterTaskAttempt(taskAttemptId1);
-    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
-    assertNull(entityTracker.getNodeIdForContainer(containerId1));
-    assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
-    assertEquals(0, entityTracker.nodeMap.size());
-    assertEquals(0, entityTracker.attemptToNodeMap.size());
-    assertEquals(0, entityTracker.containerToNodeMap.size());
-
-    // Register taskAttempt, unregister container. TaskAttempt should also be unregistered
-    ContainerId containerId201 = constructContainerId(201);
-    TezTaskAttemptID taskAttemptId201 = constructTaskAttemptId(201);
-    entityTracker.registerTaskAttempt(containerId201, taskAttemptId201, host1, port);
-    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId201));
-    assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
-
-    entityTracker.unregisterContainer(containerId201);
-    assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
-    assertNull(entityTracker.getNodeIdForContainer(containerId201));
-    assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
-    assertEquals(0, entityTracker.nodeMap.size());
-    assertEquals(0, entityTracker.attemptToNodeMap.size());
-    assertEquals(0, entityTracker.containerToNodeMap.size());
-
-    entityTracker.unregisterTaskAttempt(taskAttemptId201); // No errors
-  }
-
-
-  private ContainerId constructContainerId(int id) {
-    ContainerId containerId = mock(ContainerId.class);
-    doReturn(id).when(containerId).getId();
-    doReturn((long)id).when(containerId).getContainerId();
-    return containerId;
-  }
-
-  private TezTaskAttemptID constructTaskAttemptId(int id) {
-    TezTaskAttemptID taskAttemptId = mock(TezTaskAttemptID.class);
-    doReturn(id).when(taskAttemptId).getId();
-    return taskAttemptId;
-  }
-
-}


[11/11] hive git commit: HIVE-12934 branch-2.0 addendum. Update version fro new modules to reflect branch version. (Siddharth Seth)

Posted by ss...@apache.org.
HIVE-12934 branch-2.0 addendum. Update version fro new modules to
reflect branch version. (Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/98d51677
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/98d51677
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/98d51677

Branch: refs/heads/branch-2.0
Commit: 98d516770756df44bbf13f09cfa1cf1d7e4508d2
Parents: 9886414
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 1 09:57:26 2016 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Feb 1 09:57:26 2016 -0800

----------------------------------------------------------------------
 llap-common/pom.xml | 2 +-
 llap-tez/pom.xml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/98d51677/llap-common/pom.xml
----------------------------------------------------------------------
diff --git a/llap-common/pom.xml b/llap-common/pom.xml
index 5343479..ac675cd 100644
--- a/llap-common/pom.xml
+++ b/llap-common/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

http://git-wip-us.apache.org/repos/asf/hive/blob/98d51677/llap-tez/pom.xml
----------------------------------------------------------------------
diff --git a/llap-tez/pom.xml b/llap-tez/pom.xml
index ce020da..812d429 100644
--- a/llap-tez/pom.xml
+++ b/llap-tez/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>org.apache.hive</groupId>
     <artifactId>hive</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.0.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 


[05/11] hive git commit: HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)

Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
deleted file mode 100644
index eb6384f..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++ /dev/null
@@ -1,755 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.hive.llap.tezplugins.helpers.SourceStateTracker;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.common.security.JobTokenSecretManager;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.event.VertexStateUpdate;
-import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
-
-  private static final Logger LOG = LoggerFactory.getLogger(LlapTaskCommunicator.class);
-
-  private static final boolean isInfoEnabled = LOG.isInfoEnabled();
-  private static final boolean isDebugEnabed = LOG.isDebugEnabled();
-
-  private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
-
-  private final ConcurrentMap<QueryIdentifierProto, ByteBuffer> credentialMap;
-
-  // Tracks containerIds and taskAttemptIds, so can be kept independent of the running DAG.
-  // When DAG specific cleanup happens, it'll be better to link this to a DAG though.
-  private final EntityTracker entityTracker = new EntityTracker();
-  private final SourceStateTracker sourceStateTracker;
-  private final Set<LlapNodeId> nodesForQuery = new HashSet<>();
-
-  private LlapDaemonProtocolClientProxy communicator;
-  private long deleteDelayOnDagComplete;
-  private final LlapTaskUmbilicalProtocol umbilical;
-  private final Token<LlapTokenIdentifier> token;
-
-  // These two structures track the list of known nodes, and the list of nodes which are sending in keep-alive heartbeats.
-  // Primarily for debugging purposes a.t.m, since there's some unexplained TASK_TIMEOUTS which are currently being observed.
-  private final ConcurrentMap<LlapNodeId, Long> knownNodeMap = new ConcurrentHashMap<>();
-  private final ConcurrentMap<LlapNodeId, PingingNodeInfo> pingedNodeMap = new ConcurrentHashMap<>();
-
-
-  private volatile int currentDagId;
-  private volatile QueryIdentifierProto currentQueryIdentifierProto;
-
-  public LlapTaskCommunicator(
-      TaskCommunicatorContext taskCommunicatorContext) {
-    super(taskCommunicatorContext);
-    Credentials credentials = taskCommunicatorContext.getCredentials();
-    if (credentials != null) {
-      @SuppressWarnings("unchecked")
-      Token<LlapTokenIdentifier> llapToken =
-          (Token<LlapTokenIdentifier>)credentials.getToken(LlapTokenIdentifier.KIND_NAME);
-      this.token = llapToken;
-    } else {
-      this.token = null;
-    }
-    Preconditions.checkState((token != null) == UserGroupInformation.isSecurityEnabled());
-
-    umbilical = new LlapTaskUmbilicalProtocolImpl(getUmbilical());
-    SubmitWorkRequestProto.Builder baseBuilder = SubmitWorkRequestProto.newBuilder();
-
-    // TODO Avoid reading this from the environment
-    baseBuilder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
-    baseBuilder.setApplicationIdString(
-        taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString());
-    baseBuilder
-        .setAppAttemptNumber(taskCommunicatorContext.getApplicationAttemptId().getAttemptId());
-    baseBuilder.setTokenIdentifier(getTokenIdentifier());
-
-    BASE_SUBMIT_WORK_REQUEST = baseBuilder.build();
-
-    credentialMap = new ConcurrentHashMap<>();
-    sourceStateTracker = new SourceStateTracker(getContext(), this);
-  }
-
-  @Override
-  public void initialize() throws Exception {
-    super.initialize();
-    Configuration conf = getConf();
-    int numThreads = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS);
-    this.communicator = new LlapDaemonProtocolClientProxy(numThreads, conf, token);
-    this.deleteDelayOnDagComplete = HiveConf.getTimeVar(
-        conf, ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS);
-    LOG.info("Running LlapTaskCommunicator with "
-        + "fileCleanupDelay=" + deleteDelayOnDagComplete
-        + ", numCommunicatorThreads=" + numThreads);
-    this.communicator.init(conf);
-  }
-
-  @Override
-  public void start() {
-    super.start();
-    this.communicator.start();
-  }
-
-  @Override
-  public void shutdown() {
-    super.shutdown();
-    if (this.communicator != null) {
-      this.communicator.stop();
-    }
-  }
-
-  @Override
-  protected void startRpcServer() {
-    Configuration conf = getConf();
-    try {
-      JobTokenSecretManager jobTokenSecretManager =
-          new JobTokenSecretManager();
-      jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
-
-      int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
-          TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT);
-      server = new RPC.Builder(conf)
-          .setProtocol(LlapTaskUmbilicalProtocol.class)
-          .setBindAddress("0.0.0.0")
-          .setPort(0)
-          .setInstance(umbilical)
-          .setNumHandlers(numHandlers)
-          .setSecretManager(jobTokenSecretManager).build();
-
-      if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
-        server.refreshServiceAcl(conf, new LlapUmbilicalPolicyProvider());
-      }
-
-      server.start();
-      this.address = NetUtils.getConnectAddress(server);
-      LOG.info(
-          "Started LlapUmbilical: " + umbilical.getClass().getName() + " at address: " + address +
-              " with numHandlers=" + numHandlers);
-    } catch (IOException e) {
-      throw new TezUncheckedException(e);
-    }
-  }
-
-  @Override
-  public void registerRunningContainer(ContainerId containerId, String hostname, int port) {
-    super.registerRunningContainer(containerId, hostname, port);
-    entityTracker.registerContainer(containerId, hostname, port);
-
-  }
-
-  @Override
-  public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, String diagnostics) {
-    super.registerContainerEnd(containerId, endReason, diagnostics);
-    if (endReason == ContainerEndReason.INTERNAL_PREEMPTION) {
-      LOG.info("Processing containerEnd for container {} caused by internal preemption", containerId);
-      TezTaskAttemptID taskAttemptId = entityTracker.getTaskAttemptIdForContainer(containerId);
-      if (taskAttemptId != null) {
-        sendTaskTerminated(taskAttemptId, true);
-      }
-    }
-    entityTracker.unregisterContainer(containerId);
-  }
-
-
-  @Override
-  public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
-                                         Map<String, LocalResource> additionalResources,
-                                         Credentials credentials,
-                                         boolean credentialsChanged,
-                                         int priority)  {
-    super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
-        credentialsChanged, priority);
-    int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId();
-    if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIdentifier())) {
-      resetCurrentDag(dagId);
-    }
-
-
-    ContainerInfo containerInfo = getContainerInfo(containerId);
-    String host;
-    int port;
-    if (containerInfo != null) {
-      synchronized (containerInfo) {
-        host = containerInfo.host;
-        port = containerInfo.port;
-      }
-    } else {
-      // TODO Handle this properly
-      throw new RuntimeException("ContainerInfo not found for container: " + containerId +
-          ", while trying to launch task: " + taskSpec.getTaskAttemptID());
-    }
-
-    LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
-    registerKnownNode(nodeId);
-    entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port);
-    nodesForQuery.add(nodeId);
-
-    sourceStateTracker.registerTaskForStateUpdates(host, port, taskSpec.getInputs());
-    FragmentRuntimeInfo fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo(
-        taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority);
-    SubmitWorkRequestProto requestProto;
-
-    try {
-      requestProto = constructSubmitWorkRequest(containerId, taskSpec, fragmentRuntimeInfo);
-    } catch (IOException e) {
-      throw new RuntimeException("Failed to construct request", e);
-    }
-
-    // Have to register this up front right now. Otherwise, it's possible for the task to start
-    // sending out status/DONE/KILLED/FAILED messages before TAImpl knows how to handle them.
-    getContext()
-        .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
-    communicator.sendSubmitWork(requestProto, host, port,
-        new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
-          @Override
-          public void setResponse(SubmitWorkResponseProto response) {
-            if (response.hasSubmissionState()) {
-              LlapDaemonProtocolProtos.SubmissionStateProto ss = response.getSubmissionState();
-              if (ss.equals(LlapDaemonProtocolProtos.SubmissionStateProto.REJECTED)) {
-                LOG.info(
-                    "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
-                        containerId + ", Service Busy");
-                getContext().taskKilled(taskSpec.getTaskAttemptID(),
-                    TaskAttemptEndReason.EXECUTOR_BUSY, "Service Busy");
-                return;
-              }
-            } else {
-              // TODO: Provide support for reporting errors
-              // This should never happen as server always returns a valid status on success
-              throw new RuntimeException("SubmissionState in response is expected!");
-            }
-            LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
-          }
-
-          @Override
-          public void indicateError(Throwable t) {
-            if (t instanceof ServiceException) {
-              ServiceException se = (ServiceException) t;
-              t = se.getCause();
-            }
-            if (t instanceof RemoteException) {
-              RemoteException re = (RemoteException) t;
-              // All others from the remote service cause the task to FAIL.
-              LOG.info(
-                  "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
-                      containerId, t);
-              getContext()
-                  .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
-                      t.toString());
-            } else {
-              // Exception from the RPC layer - communication failure, consider as KILLED / service down.
-              if (t instanceof IOException) {
-                LOG.info(
-                    "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
-                        containerId + ", Communication Error");
-                getContext().taskKilled(taskSpec.getTaskAttemptID(),
-                    TaskAttemptEndReason.COMMUNICATION_ERROR, "Communication Error");
-              } else {
-                // Anything else is a FAIL.
-                LOG.info(
-                    "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
-                        containerId, t);
-                getContext()
-                    .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
-                        t.getMessage());
-              }
-            }
-          }
-        });
-  }
-
-  @Override
-  public void unregisterRunningTaskAttempt(final TezTaskAttemptID taskAttemptId,
-                                           TaskAttemptEndReason endReason,
-                                           String diagnostics) {
-    super.unregisterRunningTaskAttempt(taskAttemptId, endReason, diagnostics);
-
-    if (endReason == TaskAttemptEndReason.INTERNAL_PREEMPTION) {
-      LOG.info("Processing taskEnd for task {} caused by internal preemption", taskAttemptId);
-      sendTaskTerminated(taskAttemptId, false);
-    }
-    entityTracker.unregisterTaskAttempt(taskAttemptId);
-    // This will also be invoked for tasks which have been KILLED / rejected by the daemon.
-    // Informing the daemon becomes necessary once the LlapScheduler supports preemption
-    // and/or starts attempting to kill tasks which may be running on a node.
-  }
-
-  private void sendTaskTerminated(final TezTaskAttemptID taskAttemptId,
-                                  boolean invokedByContainerEnd) {
-    LOG.info(
-        "DBG: Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}",
-        taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd");
-    LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId);
-    // NodeId can be null if the task gets unregistered due to failure / being killed by the daemon itself
-    if (nodeId != null) {
-      TerminateFragmentRequestProto request =
-          TerminateFragmentRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifierProto)
-              .setFragmentIdentifierString(taskAttemptId.toString()).build();
-      communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(),
-          new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() {
-            @Override
-            public void setResponse(TerminateFragmentResponseProto response) {
-            }
-
-            @Override
-            public void indicateError(Throwable t) {
-              LOG.warn("Failed to send terminate fragment request for {}",
-                  taskAttemptId.toString());
-            }
-          });
-    } else {
-      LOG.info(
-          "Not sending terminate request for fragment {} since it's node is not known. Already unregistered",
-          taskAttemptId.toString());
-    }
-  }
-
-
-
-
-  @Override
-  public void dagComplete(final int dagIdentifier) {
-    QueryCompleteRequestProto request = QueryCompleteRequestProto.newBuilder()
-        .setQueryIdentifier(constructQueryIdentifierProto(dagIdentifier))
-        .setDeleteDelay(deleteDelayOnDagComplete).build();
-    for (final LlapNodeId llapNodeId : nodesForQuery) {
-      LOG.info("Sending dagComplete message for {}, to {}", dagIdentifier, llapNodeId);
-      communicator.sendQueryComplete(request, llapNodeId.getHostname(), llapNodeId.getPort(),
-          new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<LlapDaemonProtocolProtos.QueryCompleteResponseProto>() {
-            @Override
-            public void setResponse(LlapDaemonProtocolProtos.QueryCompleteResponseProto response) {
-            }
-
-            @Override
-            public void indicateError(Throwable t) {
-              LOG.warn("Failed to indicate dag complete dagId={} to node {}", dagIdentifier, llapNodeId);
-            }
-          });
-    }
-
-    nodesForQuery.clear();
-    // TODO Ideally move some of the other cleanup code from resetCurrentDag over here
-  }
-
-  @Override
-  public void onVertexStateUpdated(VertexStateUpdate vertexStateUpdate) {
-    // Delegate updates over to the source state tracker.
-    sourceStateTracker
-        .sourceStateUpdated(vertexStateUpdate.getVertexName(), vertexStateUpdate.getVertexState());
-  }
-
-  public void sendStateUpdate(final String host, final int port,
-                              final SourceStateUpdatedRequestProto request) {
-    communicator.sendSourceStateUpdate(request, host, port,
-        new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<SourceStateUpdatedResponseProto>() {
-          @Override
-          public void setResponse(SourceStateUpdatedResponseProto response) {
-          }
-
-          @Override
-          public void indicateError(Throwable t) {
-            // TODO HIVE-10280.
-            // Ideally, this should be retried for a while, after which the node should be marked as failed.
-            // Considering tasks are supposed to run fast. Failing the task immediately may be a good option.
-            LOG.error(
-                "Failed to send state update to node: " + host + ":" + port + ", StateUpdate=" +
-                    request, t);
-          }
-        });
-  }
-
-
-  private static class PingingNodeInfo {
-    final AtomicLong logTimestamp;
-    final AtomicInteger pingCount;
-
-    PingingNodeInfo(long currentTs) {
-      logTimestamp = new AtomicLong(currentTs);
-      pingCount = new AtomicInteger(1);
-    }
-  }
-
-  public void registerKnownNode(LlapNodeId nodeId) {
-    Long old = knownNodeMap.putIfAbsent(nodeId,
-        TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS));
-    if (old == null) {
-      if (isInfoEnabled) {
-        LOG.info("Added new known node: {}", nodeId);
-      }
-    }
-  }
-
-  public void registerPingingNode(LlapNodeId nodeId) {
-    long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
-    PingingNodeInfo ni = new PingingNodeInfo(currentTs);
-    PingingNodeInfo old = pingedNodeMap.put(nodeId, ni);
-    if (old == null) {
-      if (isInfoEnabled) {
-        LOG.info("Added new pinging node: [{}]", nodeId);
-      }
-    } else {
-      old.pingCount.incrementAndGet();
-    }
-    // The node should always be known by this point. Log occasionally if it is not known.
-    if (!knownNodeMap.containsKey(nodeId)) {
-      if (old == null) {
-        // First time this is seen. Log it.
-        LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, ni.pingCount.get());
-      } else {
-        // Pinged before. Log only occasionally.
-        if (currentTs > old.logTimestamp.get() + 5000l) { // 5 seconds elapsed. Log again.
-          LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, old.pingCount.get());
-          old.logTimestamp.set(currentTs);
-        }
-      }
-
-    }
-  }
-
-
-  private final AtomicLong nodeNotFoundLogTime = new AtomicLong(0);
-
-  void nodePinged(String hostname, int port) {
-    LlapNodeId nodeId = LlapNodeId.getInstance(hostname, port);
-    registerPingingNode(nodeId);
-    BiMap<ContainerId, TezTaskAttemptID> biMap =
-        entityTracker.getContainerAttemptMapForNode(nodeId);
-    if (biMap != null) {
-      synchronized (biMap) {
-        for (Map.Entry<ContainerId, TezTaskAttemptID> entry : biMap.entrySet()) {
-          getContext().taskAlive(entry.getValue());
-          getContext().containerAlive(entry.getKey());
-        }
-      }
-    } else {
-      long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
-      if (currentTs > nodeNotFoundLogTime.get() + 5000l) {
-        LOG.warn("Received ping from node without any registered tasks or containers: " + hostname +
-            ":" + port +
-            ". Could be caused by pre-emption by the AM," +
-            " or a mismatched hostname. Enable debug logging for mismatched host names");
-        nodeNotFoundLogTime.set(currentTs);
-      }
-    }
-  }
-
-  private void resetCurrentDag(int newDagId) {
-    // Working on the assumption that a single DAG runs at a time per AM.
-    currentQueryIdentifierProto = constructQueryIdentifierProto(newDagId);
-    sourceStateTracker.resetState(newDagId);
-    nodesForQuery.clear();
-    LOG.info("CurrentDagId set to: " + newDagId + ", name=" + getContext().getCurrentDagName());
-    // TODO Is it possible for heartbeats to come in from lost tasks - those should be told to die, which
-    // is likely already happening.
-  }
-
-  private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId,
-                                                            TaskSpec taskSpec,
-                                                            FragmentRuntimeInfo fragmentRuntimeInfo) throws
-      IOException {
-    SubmitWorkRequestProto.Builder builder =
-        SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
-    builder.setContainerIdString(containerId.toString());
-    builder.setAmHost(getAddress().getHostName());
-    builder.setAmPort(getAddress().getPort());
-    Credentials taskCredentials = new Credentials();
-    // Credentials can change across DAGs. Ideally construct only once per DAG.
-    taskCredentials.addAll(getContext().getCredentials());
-
-    Preconditions.checkState(currentQueryIdentifierProto.getDagIdentifier() ==
-        taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId());
-    ByteBuffer credentialsBinary = credentialMap.get(currentQueryIdentifierProto);
-    if (credentialsBinary == null) {
-      credentialsBinary = serializeCredentials(getContext().getCredentials());
-      credentialMap.putIfAbsent(currentQueryIdentifierProto, credentialsBinary.duplicate());
-    } else {
-      credentialsBinary = credentialsBinary.duplicate();
-    }
-    builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
-    builder.setFragmentSpec(Converters.convertTaskSpecToProto(taskSpec));
-    builder.setFragmentRuntimeInfo(fragmentRuntimeInfo);
-    return builder.build();
-  }
-
-  private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
-    Credentials containerCredentials = new Credentials();
-    containerCredentials.addAll(credentials);
-    DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
-    containerCredentials.writeTokenStorageToStream(containerTokens_dob);
-    return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
-  }
-
-
-
-  protected class LlapTaskUmbilicalProtocolImpl implements LlapTaskUmbilicalProtocol {
-
-    private final TezTaskUmbilicalProtocol tezUmbilical;
-
-    public LlapTaskUmbilicalProtocolImpl(TezTaskUmbilicalProtocol tezUmbilical) {
-      this.tezUmbilical = tezUmbilical;
-    }
-
-    @Override
-    public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
-      return tezUmbilical.canCommit(taskid);
-    }
-
-    @Override
-    public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
-        TezException {
-      return tezUmbilical.heartbeat(request);
-    }
-
-    @Override
-    public void nodeHeartbeat(Text hostname, int port) throws IOException {
-      nodePinged(hostname.toString(), port);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Received heartbeat from [" + hostname + ":" + port +"]");
-      }
-    }
-
-    @Override
-    public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException {
-      // TODO Unregister the task for state updates, which could in turn unregister the node.
-      getContext().taskKilled(taskAttemptId,
-          TaskAttemptEndReason.EXTERNAL_PREEMPTION, "Attempt preempted");
-      entityTracker.unregisterTaskAttempt(taskAttemptId);
-    }
-
-    @Override
-    public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
-      return versionID;
-    }
-
-    @Override
-    public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
-                                                  int clientMethodsHash) throws IOException {
-      return ProtocolSignature.getProtocolSignature(this, protocol,
-          clientVersion, clientMethodsHash);
-    }
-  }
-
-  /**
-   * Track the association between known containers and taskAttempts, along with the nodes they are assigned to.
-   */
-  @VisibleForTesting
-  static final class EntityTracker {
-    @VisibleForTesting
-    final ConcurrentMap<TezTaskAttemptID, LlapNodeId> attemptToNodeMap = new ConcurrentHashMap<>();
-    @VisibleForTesting
-    final ConcurrentMap<ContainerId, LlapNodeId> containerToNodeMap = new ConcurrentHashMap<>();
-    @VisibleForTesting
-    final ConcurrentMap<LlapNodeId, BiMap<ContainerId, TezTaskAttemptID>> nodeMap = new ConcurrentHashMap<>();
-
-    void registerTaskAttempt(ContainerId containerId, TezTaskAttemptID taskAttemptId, String host, int port) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Registering " + containerId + ", " + taskAttemptId + " for node: " + host + ":" + port);
-      }
-      LlapNodeId llapNodeId = LlapNodeId.getInstance(host, port);
-      attemptToNodeMap.putIfAbsent(taskAttemptId, llapNodeId);
-
-      registerContainer(containerId, host, port);
-
-      // nodeMap registration.
-      BiMap<ContainerId, TezTaskAttemptID> tmpMap = HashBiMap.create();
-      BiMap<ContainerId, TezTaskAttemptID> old = nodeMap.putIfAbsent(llapNodeId, tmpMap);
-      BiMap<ContainerId, TezTaskAttemptID> usedInstance;
-      usedInstance = old == null ? tmpMap : old;
-      synchronized(usedInstance) {
-        usedInstance.put(containerId, taskAttemptId);
-      }
-      // Make sure to put the instance back again, in case it was removed as part of a
-      // containerEnd/taskEnd invocation.
-      nodeMap.putIfAbsent(llapNodeId, usedInstance);
-    }
-
-    void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
-      LlapNodeId llapNodeId = attemptToNodeMap.remove(attemptId);
-      if (llapNodeId == null) {
-        // Possible since either container / task can be unregistered.
-        return;
-      }
-
-      BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
-      ContainerId matched = null;
-      if (bMap != null) {
-        synchronized(bMap) {
-          matched = bMap.inverse().remove(attemptId);
-        }
-        if (bMap.isEmpty()) {
-          nodeMap.remove(llapNodeId);
-        }
-      }
-
-      // Remove the container mapping
-      if (matched != null) {
-        containerToNodeMap.remove(matched);
-      }
-
-    }
-
-    void registerContainer(ContainerId containerId, String hostname, int port) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Registering " + containerId + " for node: " + hostname + ":" + port);
-      }
-      containerToNodeMap.putIfAbsent(containerId, LlapNodeId.getInstance(hostname, port));
-      // nodeMap registration is not required, since there's no taskId association.
-    }
-
-    LlapNodeId getNodeIdForContainer(ContainerId containerId) {
-      return containerToNodeMap.get(containerId);
-    }
-
-    LlapNodeId getNodeIdForTaskAttempt(TezTaskAttemptID taskAttemptId) {
-      return attemptToNodeMap.get(taskAttemptId);
-    }
-
-    ContainerId getContainerIdForAttempt(TezTaskAttemptID taskAttemptId) {
-      LlapNodeId llapNodeId = getNodeIdForTaskAttempt(taskAttemptId);
-      if (llapNodeId != null) {
-        BiMap<TezTaskAttemptID, ContainerId> bMap = nodeMap.get(llapNodeId).inverse();
-        if (bMap != null) {
-          synchronized (bMap) {
-            return bMap.get(taskAttemptId);
-          }
-        } else {
-          return null;
-        }
-      } else {
-        return null;
-      }
-    }
-
-    TezTaskAttemptID getTaskAttemptIdForContainer(ContainerId containerId) {
-      LlapNodeId llapNodeId = getNodeIdForContainer(containerId);
-      if (llapNodeId != null) {
-        BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
-        if (bMap != null) {
-          synchronized (bMap) {
-            return bMap.get(containerId);
-          }
-        } else {
-          return null;
-        }
-      } else {
-        return null;
-      }
-    }
-
-    void unregisterContainer(ContainerId containerId) {
-      LlapNodeId llapNodeId = containerToNodeMap.remove(containerId);
-      if (llapNodeId == null) {
-        // Possible since either container / task can be unregistered.
-        return;
-      }
-
-      BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
-      TezTaskAttemptID matched = null;
-      if (bMap != null) {
-        synchronized(bMap) {
-          matched = bMap.remove(containerId);
-        }
-        if (bMap.isEmpty()) {
-          nodeMap.remove(llapNodeId);
-        }
-      }
-
-      // Remove the container mapping
-      if (matched != null) {
-        attemptToNodeMap.remove(matched);
-      }
-    }
-
-    /**
-     * Return a {@link BiMap} containing container->taskAttemptId mapping for the host specified.
-     * </p>
-     * <p/>
-     * This method return the internal structure used by the EntityTracker. Users must synchronize
-     * on the structure to ensure correct usage.
-     *
-     * @param llapNodeId
-     * @return
-     */
-    BiMap<ContainerId, TezTaskAttemptID> getContainerAttemptMapForNode(LlapNodeId llapNodeId) {
-      BiMap<ContainerId, TezTaskAttemptID> biMap = nodeMap.get(llapNodeId);
-      return biMap;
-    }
-
-  }
-
-  private QueryIdentifierProto constructQueryIdentifierProto(int dagIdentifier) {
-    return QueryIdentifierProto.newBuilder()
-        .setAppIdentifier(getContext().getCurrentAppIdentifier()).setDagIdentifier(dagIdentifier)
-        .build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
deleted file mode 100644
index 4102f5b..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
-import org.apache.hadoop.security.authorize.Service;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
-
-public class LlapUmbilicalPolicyProvider extends TezAMPolicyProvider {
-
-  private static Service[] services;
-  private static final Object servicesLock = new Object();
-
-  @Override
-  public Service[] getServices() {
-    if (services != null) return services;
-    synchronized (servicesLock) {
-      if (services != null) return services;
-      Service[] parentSvc = super.getServices();
-      Service[] result = Arrays.copyOf(parentSvc, parentSvc.length + 1);
-      result[parentSvc.length] =  new Service(
-          TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL,
-          LlapTaskUmbilicalProtocol.class);
-      return (services = result);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
deleted file mode 100644
index fded9bf..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins.helpers;
-
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
-import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.tezplugins.Converters;
-import org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator;
-import org.apache.tez.dag.api.event.VertexState;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.input.MultiMRInput;
-import org.apache.tez.runtime.api.impl.InputSpec;
-
-public class SourceStateTracker {
-
-  private static final Logger LOG = LoggerFactory.getLogger(SourceStateTracker.class);
-
-  private final TaskCommunicatorContext taskCommunicatorContext;
-  private final LlapTaskCommunicator taskCommunicator;
-
-  private final QueryIdentifierProto BASE_QUERY_IDENTIFIER;
-
-  // Tracks vertices for which notifications have been registered
-  private final Set<String> notificationRegisteredVertices = new HashSet<>();
-
-  private final Map<String, SourceInfo> sourceInfoMap = new HashMap<>();
-  private final Map<LlapNodeId, NodeInfo> nodeInfoMap = new HashMap<>();
-
-  private volatile QueryIdentifierProto currentQueryIdentifier;
-
-  public SourceStateTracker(TaskCommunicatorContext taskCommunicatorContext,
-                            LlapTaskCommunicator taskCommunicator) {
-    this.taskCommunicatorContext = taskCommunicatorContext;
-    this.taskCommunicator = taskCommunicator;
-    BASE_QUERY_IDENTIFIER = QueryIdentifierProto.newBuilder()
-        .setAppIdentifier(taskCommunicatorContext.getCurrentAppIdentifier()).build();
-  }
-
-  /**
-   * To be invoked after each DAG completes.
-   */
-  public synchronized void resetState(int newDagId) {
-    sourceInfoMap.clear();
-    nodeInfoMap.clear();
-    notificationRegisteredVertices.clear();
-    this.currentQueryIdentifier =
-        QueryIdentifierProto.newBuilder(BASE_QUERY_IDENTIFIER).setDagIdentifier(newDagId).build();
-  }
-
-  /**
-   * Used to register a task for state updates. Effectively registers for state updates to go to the specific node.
-   * @param host
-   * @param port
-   * @param inputSpecList
-   */
-  public synchronized void registerTaskForStateUpdates(String host, int port,
-                                                       List<InputSpec> inputSpecList) {
-
-    // Add tracking information. Check if source state already known and send out an update if it is.
-
-    List<String> sourcesOfInterest = getSourceInterestList(inputSpecList);
-    if (sourcesOfInterest != null && !sourcesOfInterest.isEmpty()) {
-      LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
-      NodeInfo nodeInfo = getNodeInfo(nodeId);
-
-      // Set up the data structures, before any notifications come in.
-      for (String src : sourcesOfInterest) {
-        VertexState oldStateForNode = nodeInfo.getLastKnownStateForSource(src);
-        if (oldStateForNode == null) {
-          // Not registered for this node.
-          // Register and send state if it is successful.
-          SourceInfo srcInfo = getSourceInfo(src);
-          srcInfo.addNode(nodeId);
-
-          nodeInfo.addSource(src, srcInfo.lastKnownState);
-          if (srcInfo.lastKnownState == VertexState.SUCCEEDED) {
-            sendStateUpdateToNode(nodeId, src, srcInfo.lastKnownState);
-          }
-
-        } else {
-          // Already registered to send updates to this node for the specific source.
-          // Nothing to do for now, unless tracking tasks at a later point.
-        }
-
-        // Setup for actual notifications, if not already done for a previous task.
-        maybeRegisterForVertexUpdates(src);
-      }
-    } else {
-      // Don't need to track anything for this task. No new notifications, etc.
-    }
-  }
-
-  /**
-   * Handled notifications on state updates for sources
-   * @param sourceName
-   * @param sourceState
-   */
-  public synchronized void sourceStateUpdated(String sourceName, VertexState sourceState) {
-    SourceInfo sourceInfo = getSourceInfo(sourceName);
-    // Update source info if the state is SUCCEEDED
-    if (sourceState == VertexState.SUCCEEDED) {
-      sourceInfo.numCompletedTasks = taskCommunicatorContext.getVertexCompletedTaskCount(sourceName);
-      sourceInfo.numTasks = taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
-    }
-    sourceInfo.lastKnownState = sourceState;
-    // Checking state per node for future failure handling scenarios, where an update
-    // to a single node may fail.
-    for (LlapNodeId nodeId : sourceInfo.getInterestedNodes()) {
-      NodeInfo nodeInfo = nodeInfoMap.get(nodeId);
-      VertexState lastStateForNode = nodeInfo.getLastKnownStateForSource(sourceName);
-      // Send only if the state has changed.
-      if (lastStateForNode != sourceState) {
-        nodeInfo.setLastKnownStateForSource(sourceName, sourceState);
-        sendStateUpdateToNode(nodeId, sourceName, sourceState);
-      }
-    }
-  }
-
-
-  // Assumes serialized DAGs within an AM, and a reset of structures after each DAG completes.
-  /**
-   * Constructs FragmentRuntimeInfo for scheduling within LLAP daemons.
-   * Also caches state based on state updates.
-   * @param vertexName
-   * @param fragmentNumber
-   * @param priority
-   * @return
-   */
-  public synchronized FragmentRuntimeInfo getFragmentRuntimeInfo(String vertexName, int fragmentNumber,
-                                                                 int priority) {
-    FragmentRuntimeInfo.Builder builder = FragmentRuntimeInfo.newBuilder();
-    maybeRegisterForVertexUpdates(vertexName);
-
-    MutableInt totalTaskCount = new MutableInt(0);
-    MutableInt completedTaskCount = new MutableInt(0);
-    computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, vertexName);
-
-    builder.setNumSelfAndUpstreamCompletedTasks(completedTaskCount.intValue());
-    builder.setNumSelfAndUpstreamTasks(totalTaskCount.intValue());
-    builder.setDagStartTime(taskCommunicatorContext.getDagStartTime());
-    builder.setWithinDagPriority(priority);
-    builder.setFirstAttemptStartTime(taskCommunicatorContext.getFirstAttemptStartTime(vertexName, fragmentNumber));
-    builder.setCurrentAttemptStartTime(System.currentTimeMillis());
-    return builder.build();
-  }
-
-  private void computeUpstreamTaskCounts(MutableInt completedTaskCount, MutableInt totalTaskCount, String sourceName) {
-    SourceInfo sourceInfo = getSourceInfo(sourceName);
-    if (sourceInfo.lastKnownState == VertexState.SUCCEEDED) {
-      // Some of the information in the source is complete. Don't need to fetch it from the context.
-      completedTaskCount.add(sourceInfo.numCompletedTasks);
-      totalTaskCount.add(sourceInfo.numTasks);
-    } else {
-      completedTaskCount.add(taskCommunicatorContext.getVertexCompletedTaskCount(sourceName));
-      int totalCount =taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
-      // Uninitialized vertices will report count as 0.
-      totalCount = totalCount == -1 ? 0 : totalCount;
-      totalTaskCount.add(totalCount);
-    }
-
-    // Walk through all the source vertices
-    for (String up : taskCommunicatorContext.getInputVertexNames(sourceName)) {
-      computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, up);
-    }
-  }
-
-  private static class SourceInfo {
-
-    // Always start in the running state. Requests for state updates will be sent out after registration.
-    private VertexState lastKnownState = VertexState.RUNNING;
-
-    // Used for sending notifications about a vertex completed. For canFinish
-    // Can be converted to a Tez event, if this is sufficient to decide on pre-emption
-    private final List<LlapNodeId> interestedNodes = new LinkedList<>();
-
-    // Used for sending information for scheduling priority.
-    private int numTasks;
-    private int numCompletedTasks;
-
-    void addNode(LlapNodeId nodeId) {
-      interestedNodes.add(nodeId);
-    }
-
-    List<LlapNodeId> getInterestedNodes() {
-      return this.interestedNodes;
-    }
-
-
-
-  }
-
-  private synchronized SourceInfo getSourceInfo(String srcName) {
-    SourceInfo sourceInfo = sourceInfoMap.get(srcName);
-    if (sourceInfo == null) {
-      sourceInfo = new SourceInfo();
-      sourceInfoMap.put(srcName, sourceInfo);
-    }
-    return sourceInfo;
-  }
-
-
-  private static class NodeInfo {
-    private final Map<String, VertexState> sourcesOfInterest = new HashMap<>();
-
-    void addSource(String srcName, VertexState sourceState) {
-      sourcesOfInterest.put(srcName, sourceState);
-    }
-
-    VertexState getLastKnownStateForSource(String src) {
-      return sourcesOfInterest.get(src);
-    }
-
-    void setLastKnownStateForSource(String src, VertexState state) {
-      sourcesOfInterest.put(src, state);
-    }
-  }
-
-  private synchronized NodeInfo getNodeInfo(LlapNodeId llapNodeId) {
-    NodeInfo nodeInfo = nodeInfoMap.get(llapNodeId);
-    if (nodeInfo == null) {
-      nodeInfo = new NodeInfo();
-      nodeInfoMap.put(llapNodeId, nodeInfo);
-    }
-    return nodeInfo;
-  }
-
-
-  private List<String> getSourceInterestList(List<InputSpec> inputSpecList) {
-    List<String> sourcesOfInterest = Collections.emptyList();
-    if (inputSpecList != null) {
-      boolean alreadyFound = false;
-      for (InputSpec inputSpec : inputSpecList) {
-        if (isSourceOfInterest(inputSpec)) {
-          if (!alreadyFound) {
-            alreadyFound = true;
-            sourcesOfInterest = new LinkedList<>();
-          }
-          sourcesOfInterest.add(inputSpec.getSourceVertexName());
-        }
-      }
-    }
-    return sourcesOfInterest;
-  }
-
-
-  private void maybeRegisterForVertexUpdates(String sourceName) {
-    if (!notificationRegisteredVertices.contains(sourceName)) {
-      notificationRegisteredVertices.add(sourceName);
-      taskCommunicatorContext.registerForVertexStateUpdates(sourceName, EnumSet.of(
-          VertexState.RUNNING, VertexState.SUCCEEDED));
-    }
-  }
-
-  private boolean isSourceOfInterest(InputSpec inputSpec) {
-    String inputClassName = inputSpec.getInputDescriptor().getClassName();
-    // MRInput is not of interest since it'll always be ready.
-    return !(inputClassName.equals(MRInputLegacy.class.getName()) || inputClassName.equals(
-        MultiMRInput.class.getName()) || inputClassName.equals(MRInput.class.getName()));
-  }
-
-  void sendStateUpdateToNode(LlapNodeId nodeId, String sourceName, VertexState state) {
-    taskCommunicator.sendStateUpdate(nodeId.getHostname(), nodeId.getPort(),
-        SourceStateUpdatedRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifier)
-            .setSrcName(sourceName).setState(Converters.fromVertexState(state)).build());
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java b/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java
deleted file mode 100644
index 89b7198..0000000
--- a/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-class ContainerFactory {
-  final ApplicationAttemptId customAppAttemptId;
-  AtomicLong nextId;
-
-  public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) {
-    this.nextId = new AtomicLong(1);
-    ApplicationId appId =
-        ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId());
-    this.customAppAttemptId =
-        ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId());
-  }
-
-  public Container createContainer(Resource capability, Priority priority, String hostname,
-      int port) {
-    ContainerId containerId =
-        ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement());
-    NodeId nodeId = NodeId.newInstance(hostname, port);
-    String nodeHttpAddress = "hostname:0"; // TODO: include UI ports
-
-    Container container =
-        Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null);
-
-    return container;
-  }
-}