You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ss...@apache.org on 2016/02/01 18:58:00 UTC
[01/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 29a988fd9 -> 98d516770
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
new file mode 100644
index 0000000..36d8ffd
--- /dev/null
+++ b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskSchedulerService.java
@@ -0,0 +1,684 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl;
+import org.apache.hadoop.hive.llap.testhelpers.ControlledClock;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestLlapTaskSchedulerService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestLlapTaskSchedulerService.class);
+
+ private static final String HOST1 = "host1";
+ private static final String HOST2 = "host2";
+ private static final String HOST3 = "host3";
+
+ @Test (timeout = 5000)
+ public void testSimpleLocalAllocation() throws IOException, InterruptedException {
+
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
+
+ try {
+ Priority priority1 = Priority.newInstance(1);
+ String[] hosts1 = new String[]{HOST1};
+
+ Object task1 = new Object();
+ Object clientCookie1 = new Object();
+
+ tsWrapper.controlScheduler(true);
+ tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
+
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+
+ verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
+ // TODO Verify this is on host1.
+ assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+ @Test (timeout = 5000)
+ public void testSimpleNoLocalityAllocation() throws IOException, InterruptedException {
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
+
+ try {
+ Priority priority1 = Priority.newInstance(1);
+
+ Object task1 = new Object();
+ Object clientCookie1 = new Object();
+ tsWrapper.controlScheduler(true);
+ tsWrapper.allocateTask(task1, null, priority1, clientCookie1);
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
+ assertEquals(1, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+
+ @Test(timeout=5000)
+ public void testPreemption() throws InterruptedException, IOException {
+
+ Priority priority1 = Priority.newInstance(1);
+ Priority priority2 = Priority.newInstance(2);
+ String [] hosts = new String[] {HOST1};
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1);
+ try {
+
+ Object task1 = "task1";
+ Object clientCookie1 = "cookie1";
+ Object task2 = "task2";
+ Object clientCookie2 = "cookie2";
+ Object task3 = "task3";
+ Object clientCookie3 = "cookie3";
+ Object task4 = "task4";
+ Object clientCookie4 = "cookie4";
+
+ tsWrapper.controlScheduler(true);
+ tsWrapper.allocateTask(task1, hosts, priority2, clientCookie1);
+ tsWrapper.allocateTask(task2, hosts, priority2, clientCookie2);
+ tsWrapper.allocateTask(task3, hosts, priority2, clientCookie3);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(any(Object.class),
+ any(Object.class), any(Container.class));
+ assertEquals(2, tsWrapper.ts.dagStats.numLocalAllocations);
+ assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+
+ reset(tsWrapper.mockAppCallback);
+
+ tsWrapper.allocateTask(task4, hosts, priority1, clientCookie4);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
+
+
+ tsWrapper.deallocateTask(task2, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
+
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
+ eq(clientCookie4), any(Container.class));
+
+ } finally {
+ tsWrapper.shutdown();
+ }
+
+ }
+
+ @Test(timeout=5000)
+ public void testNodeDisabled() throws IOException, InterruptedException {
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(10000l);
+ try {
+ Priority priority1 = Priority.newInstance(1);
+ String[] hosts1 = new String[]{HOST1};
+ Object task1 = new Object();
+ Object clientCookie1 = new Object();
+ tsWrapper.controlScheduler(true);
+ tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 1) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1),
+ any(Container.class));
+ assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
+ assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+ assertEquals(0, tsWrapper.ts.dagStats.numNonLocalAllocations);
+ assertEquals(1, tsWrapper.ts.dagStats.numTotalAllocations);
+
+ tsWrapper.resetAppCallback();
+
+ tsWrapper.clock.setTime(10000l);
+ tsWrapper.rejectExecution(task1);
+
+ // Verify that the node is blacklisted
+ assertEquals(1, tsWrapper.ts.dagStats.numRejectedTasks);
+ assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
+ LlapTaskSchedulerService.NodeInfo disabledNodeInfo = tsWrapper.ts.disabledNodesQueue.peek();
+ assertNotNull(disabledNodeInfo);
+ assertEquals(HOST1, disabledNodeInfo.serviceInstance.getHost());
+ assertEquals((10000l), disabledNodeInfo.getDelay(TimeUnit.MILLISECONDS));
+ assertEquals((10000l + 10000l), disabledNodeInfo.expireTimeMillis);
+
+ Object task2 = new Object();
+ Object clientCookie2 = new Object();
+ tsWrapper.allocateTask(task2, hosts1, priority1, clientCookie2);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback).taskAllocated(eq(task2), eq(clientCookie2), any(Container.class));
+ assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
+ assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+ assertEquals(1, tsWrapper.ts.dagStats.numNonLocalAllocations);
+ assertEquals(2, tsWrapper.ts.dagStats.numTotalAllocations);
+
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+ @Test(timeout=5000)
+ public void testNodeReEnabled() throws InterruptedException, IOException {
+ // Based on actual timing.
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(1000l);
+ try {
+ Priority priority1 = Priority.newInstance(1);
+ String[] hosts1 = new String[]{HOST1};
+ String[] hosts2 = new String[]{HOST2};
+ String[] hosts3 = new String[]{HOST3};
+
+ Object task1 = new Object();
+ Object clientCookie1 = new Object();
+ Object task2 = new Object();
+ Object clientCookie2 = new Object();
+ Object task3 = new Object();
+ Object clientCookie3 = new Object();
+
+ tsWrapper.controlScheduler(true);
+ tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
+ tsWrapper.allocateTask(task2, hosts2, priority1, clientCookie2);
+ tsWrapper.allocateTask(task3, hosts3, priority1, clientCookie3);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), any(Container.class));
+ assertEquals(3, tsWrapper.ts.dagStats.numLocalAllocations);
+ assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+ assertEquals(3, tsWrapper.ts.dagStats.numTotalAllocations);
+
+ tsWrapper.resetAppCallback();
+
+ tsWrapper.rejectExecution(task1);
+ tsWrapper.rejectExecution(task2);
+ tsWrapper.rejectExecution(task3);
+
+ // Verify that the node is blacklisted
+ assertEquals(3, tsWrapper.ts.dagStats.numRejectedTasks);
+ assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
+ assertEquals(3, tsWrapper.ts.disabledNodesQueue.size());
+
+
+ Object task4 = new Object();
+ Object clientCookie4 = new Object();
+ Object task5 = new Object();
+ Object clientCookie5 = new Object();
+ Object task6 = new Object();
+ Object clientCookie6 = new Object();
+ tsWrapper.allocateTask(task4, hosts1, priority1, clientCookie4);
+ tsWrapper.allocateTask(task5, hosts2, priority1, clientCookie5);
+ tsWrapper.allocateTask(task6, hosts3, priority1, clientCookie6);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 6) {
+ break;
+ }
+ }
+
+ ArgumentCaptor<Container> argumentCaptor = ArgumentCaptor.forClass(Container.class);
+ verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), argumentCaptor.capture());
+
+ // which affects the locality matching
+ assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
+ assertEquals(6, tsWrapper.ts.dagStats.numTotalAllocations);
+
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+ @Test (timeout = 5000)
+ public void testForceLocalityTest1() throws IOException, InterruptedException {
+ // 2 hosts. 2 per host. 5 requests at the same priority.
+ // First 3 on host1, Next at host2, Last with no host.
+ // Third request on host1 should not be allocated immediately.
+ forceLocalityTest1(true);
+
+ }
+
+ @Test (timeout = 5000)
+ public void testNoForceLocalityCounterTest1() throws IOException, InterruptedException {
+ // 2 hosts. 2 per host. 5 requests at the same priority.
+ // First 3 on host1, Next at host2, Last with no host.
+ // Third should allocate on host2, 4th on host2, 5th will wait.
+
+ forceLocalityTest1(false);
+ }
+
+ private void forceLocalityTest1(boolean forceLocality) throws IOException, InterruptedException {
+ Priority priority1 = Priority.newInstance(1);
+
+ String[] hosts = new String[] {HOST1, HOST2};
+
+ String[] hostsH1 = new String[] {HOST1};
+ String[] hostsH2 = new String[] {HOST2};
+
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, (forceLocality ? -1l : 0l));
+
+ try {
+ Object task1 = "task1";
+ Object clientCookie1 = "cookie1";
+ Object task2 = "task2";
+ Object clientCookie2 = "cookie2";
+ Object task3 = "task3";
+ Object clientCookie3 = "cookie3";
+ Object task4 = "task4";
+ Object clientCookie4 = "cookie4";
+ Object task5 = "task5";
+ Object clientCookie5 = "cookie5";
+
+ tsWrapper.controlScheduler(true);
+ //H1 - should allocate
+ tsWrapper.allocateTask(task1, hostsH1, priority1, clientCookie1);
+ //H1 - should allocate
+ tsWrapper.allocateTask(task2, hostsH1, priority1, clientCookie2);
+ //H1 - no capacity if force, should allocate otherwise
+ tsWrapper.allocateTask(task3, hostsH1, priority1, clientCookie3);
+ //H2 - should allocate
+ tsWrapper.allocateTask(task4, hostsH2, priority1, clientCookie4);
+ //No location - should allocate if force, no capacity otherwise
+ tsWrapper.allocateTask(task5, null, priority1, clientCookie5);
+
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 4) {
+ break;
+ }
+ }
+
+ // Verify no preemption requests - since everything is at the same priority
+ verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
+ ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
+ verify(tsWrapper.mockAppCallback, times(4)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+ assertEquals(4, argumentCaptor.getAllValues().size());
+ assertEquals(task1, argumentCaptor.getAllValues().get(0));
+ assertEquals(task2, argumentCaptor.getAllValues().get(1));
+ if (forceLocality) {
+ // task3 not allocated
+ assertEquals(task4, argumentCaptor.getAllValues().get(2));
+ assertEquals(task5, argumentCaptor.getAllValues().get(3));
+ } else {
+ assertEquals(task3, argumentCaptor.getAllValues().get(2));
+ assertEquals(task4, argumentCaptor.getAllValues().get(3));
+ }
+
+ //Complete one task on host1.
+ tsWrapper.deallocateTask(task1, true, null);
+
+ reset(tsWrapper.mockAppCallback);
+
+ // Try scheduling again.
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 5) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
+ argumentCaptor = ArgumentCaptor.forClass(Object.class);
+ verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+ assertEquals(1, argumentCaptor.getAllValues().size());
+ if (forceLocality) {
+ assertEquals(task3, argumentCaptor.getAllValues().get(0));
+ } else {
+ assertEquals(task5, argumentCaptor.getAllValues().get(0));
+ }
+
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testForcedLocalityUnknownHost() throws IOException, InterruptedException {
+ Priority priority1 = Priority.newInstance(1);
+
+ String[] hostsKnown = new String[]{HOST1};
+ String[] hostsUnknown = new String[]{HOST2};
+
+ TestTaskSchedulerServiceWrapper tsWrapper =
+ new TestTaskSchedulerServiceWrapper(2000, hostsKnown, 1, 1, -1l);
+ try {
+ Object task1 = "task1";
+ Object clientCookie1 = "cookie1";
+
+ Object task2 = "task2";
+ Object clientCookie2 = "cookie2";
+
+ tsWrapper.controlScheduler(true);
+ // Should allocate since H2 is not known.
+ tsWrapper.allocateTask(task1, hostsUnknown, priority1, clientCookie1);
+ tsWrapper.allocateTask(task2, hostsKnown, priority1, clientCookie2);
+
+
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
+ break;
+ }
+ }
+
+ ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
+ verify(tsWrapper.mockAppCallback, times(2))
+ .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+ assertEquals(2, argumentCaptor.getAllValues().size());
+ assertEquals(task1, argumentCaptor.getAllValues().get(0));
+ assertEquals(task2, argumentCaptor.getAllValues().get(1));
+
+
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+
+ @Test(timeout = 5000)
+ public void testForcedLocalityPreemption() throws IOException, InterruptedException {
+ Priority priority1 = Priority.newInstance(1);
+ Priority priority2 = Priority.newInstance(2);
+ String [] hosts = new String[] {HOST1, HOST2};
+
+ String [] hostsH1 = new String[] {HOST1};
+ String [] hostsH2 = new String[] {HOST2};
+
+ TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, -1l);
+
+ // Fill up host1 with p2 tasks.
+ // Leave host2 empty
+ // Try running p1 task on host1 - should preempt
+
+ try {
+ Object task1 = "task1";
+ Object clientCookie1 = "cookie1";
+ Object task2 = "task2";
+ Object clientCookie2 = "cookie2";
+ Object task3 = "task3";
+ Object clientCookie3 = "cookie3";
+ Object task4 = "task4";
+ Object clientCookie4 = "cookie4";
+
+ tsWrapper.controlScheduler(true);
+ tsWrapper.allocateTask(task1, hostsH1, priority2, clientCookie1);
+ tsWrapper.allocateTask(task2, hostsH1, priority2, clientCookie2);
+ // This request at a lower priority should not affect anything.
+ tsWrapper.allocateTask(task3, hostsH1, priority2, clientCookie3);
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
+ break;
+ }
+ }
+
+ verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
+ ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
+ verify(tsWrapper.mockAppCallback, times(2))
+ .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
+ assertEquals(2, argumentCaptor.getAllValues().size());
+ assertEquals(task1, argumentCaptor.getAllValues().get(0));
+ assertEquals(task2, argumentCaptor.getAllValues().get(1));
+
+ reset(tsWrapper.mockAppCallback);
+ // Allocate t4 at higher priority. t3 should not be allocated,
+ // and a preemption should be attempted on host1, despite host2 having available capacity
+ tsWrapper.allocateTask(task4, hostsH1, priority1, clientCookie4);
+
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
+
+ tsWrapper.deallocateTask(task1, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
+
+ while (true) {
+ tsWrapper.signalSchedulerRun();
+ tsWrapper.awaitSchedulerRun();
+ if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
+ break;
+ }
+ }
+ verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
+ eq(clientCookie4), any(Container.class));
+
+ } finally {
+ tsWrapper.shutdown();
+ }
+ }
+
+ private static class TestTaskSchedulerServiceWrapper {
+ static final Resource resource = Resource.newInstance(1024, 1);
+ Configuration conf;
+ TaskSchedulerContext mockAppCallback = mock(TaskSchedulerContext.class);
+ ControlledClock clock = new ControlledClock(new SystemClock());
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1000, 1), 1);
+ LlapTaskSchedulerServiceForTest ts;
+
+ TestTaskSchedulerServiceWrapper() throws IOException, InterruptedException {
+ this(2000l);
+ }
+
+ TestTaskSchedulerServiceWrapper(long disableTimeoutMillis) throws IOException,
+ InterruptedException {
+ this(disableTimeoutMillis, new String[]{HOST1, HOST2, HOST3}, 4,
+ ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.defaultIntVal);
+ }
+
+ TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize) throws
+ IOException, InterruptedException {
+ this(disableTimeoutMillis, hosts, numExecutors, waitQueueSize, 0l);
+ }
+
+ TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize, long localityDelayMs) throws
+ IOException, InterruptedException {
+ conf = new Configuration();
+ conf.setStrings(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, hosts);
+ conf.setInt(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, numExecutors);
+ conf.setInt(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname, waitQueueSize);
+ conf.set(ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS.varname,
+ disableTimeoutMillis + "ms");
+ conf.setBoolean(LlapFixedRegistryImpl.FIXED_REGISTRY_RESOLVE_HOST_NAMES, false);
+ conf.setLong(ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY.varname, localityDelayMs);
+
+ doReturn(appAttemptId).when(mockAppCallback).getApplicationAttemptId();
+ doReturn(11111l).when(mockAppCallback).getCustomClusterIdentifier();
+ UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
+ doReturn(userPayload).when(mockAppCallback).getInitialUserPayload();
+
+ ts = new LlapTaskSchedulerServiceForTest(mockAppCallback, clock);
+
+ controlScheduler(true);
+ ts.initialize();
+ ts.start();
+ // One scheduler pass from the nodes that are added at startup
+ signalSchedulerRun();
+ controlScheduler(false);
+ awaitSchedulerRun();
+ }
+
+ void controlScheduler(boolean val) {
+ ts.forTestsetControlScheduling(val);
+ }
+
+ void signalSchedulerRun() throws InterruptedException {
+ ts.forTestSignalSchedulingRun();
+ }
+
+ void awaitSchedulerRun() throws InterruptedException {
+ ts.forTestAwaitSchedulingRun();
+ }
+ void resetAppCallback() {
+ reset(mockAppCallback);
+ }
+
+ void shutdown() {
+ ts.shutdown();
+ }
+
+ void allocateTask(Object task, String[] hosts, Priority priority, Object clientCookie) {
+ ts.allocateTask(task, resource, hosts, null, priority, null, clientCookie);
+ }
+
+ void deallocateTask(Object task, boolean succeeded, TaskAttemptEndReason endReason) {
+ ts.deallocateTask(task, succeeded, endReason, null);
+ }
+
+ void rejectExecution(Object task) {
+ ts.deallocateTask(task, false, TaskAttemptEndReason.EXECUTOR_BUSY, null);
+ }
+ }
+
+ private static class LlapTaskSchedulerServiceForTest extends LlapTaskSchedulerService {
+
+ private AtomicBoolean controlScheduling = new AtomicBoolean(false);
+ private final Lock testLock = new ReentrantLock();
+ private final Condition schedulingCompleteCondition = testLock.newCondition();
+ private boolean schedulingComplete = false;
+ private final Condition triggerSchedulingCondition = testLock.newCondition();
+ private boolean schedulingTriggered = false;
+ private final AtomicInteger numSchedulerRuns = new AtomicInteger(0);
+
+
+ public LlapTaskSchedulerServiceForTest(
+ TaskSchedulerContext appClient, Clock clock) {
+ super(appClient, clock);
+ }
+
+ @Override
+ protected void schedulePendingTasks() {
+ testLock.lock();
+ try {
+ if (controlScheduling.get()) {
+ while (!schedulingTriggered) {
+ try {
+ triggerSchedulingCondition.await();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ numSchedulerRuns.incrementAndGet();
+ super.schedulePendingTasks();
+ schedulingTriggered = false;
+ schedulingComplete = true;
+ schedulingCompleteCondition.signal();
+ } finally {
+ testLock.unlock();
+ }
+ }
+
+ // Enable or disable test scheduling control.
+ void forTestsetControlScheduling(boolean control) {
+ this.controlScheduling.set(control);
+ }
+
+ void forTestSignalSchedulingRun() throws InterruptedException {
+ testLock.lock();
+ try {
+ schedulingTriggered = true;
+ triggerSchedulingCondition.signal();
+ } finally {
+ testLock.unlock();
+ }
+ }
+
+ void forTestAwaitSchedulingRun() throws InterruptedException {
+ testLock.lock();
+ try {
+ while (!schedulingComplete) {
+ schedulingCompleteCondition.await();
+ }
+ schedulingComplete = false;
+ } finally {
+ testLock.unlock();
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d89d64c..3d8b332 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,8 +48,10 @@
<module>ql</module>
<module>serde</module>
<module>service</module>
- <module>llap-server</module>
+ <module>llap-common</module>
<module>llap-client</module>
+ <module>llap-tez</module>
+ <module>llap-server</module>
<module>shims</module>
<module>spark-client</module>
<module>storage-api</module>
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index 0745ff3..6b596d5 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -68,6 +68,11 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-tez</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -601,6 +606,55 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 0bc6e2a..6a7d035 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.hive.ql.exec.tez;
import java.util.Collection;
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -44,7 +42,6 @@ import java.util.concurrent.TimeoutException;
import javax.security.auth.login.LoginException;
-import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.fs.FileStatus;
@@ -53,9 +50,14 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.impl.LlapProtocolClientImpl;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
import org.apache.hadoop.hive.llap.security.LlapTokenProvider;
+import org.apache.hadoop.hive.llap.tez.LlapProtocolClientProxy;
+import org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -88,9 +90,9 @@ public class TezSessionState {
private static final Logger LOG = LoggerFactory.getLogger(TezSessionState.class.getName());
private static final String TEZ_DIR = "_tez_session_dir";
public static final String LLAP_SERVICE = "LLAP";
- private static final String LLAP_SCHEDULER = "org.apache.tez.dag.app.rm.LlapTaskSchedulerService";
- private static final String LLAP_LAUNCHER = "org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher";
- private static final String LLAP_TASK_COMMUNICATOR = "org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator";
+ private static final String LLAP_SCHEDULER = LlapTaskSchedulerService.class.getName();
+ private static final String LLAP_LAUNCHER = LlapContainerLauncher.class.getName();
+ private static final String LLAP_TASK_COMMUNICATOR = LlapTaskCommunicator.class.getName();
private HiveConf conf;
private Path tezScratchDir;
@@ -251,27 +253,11 @@ public class TezSessionState {
// add configs for llap-daemon-site.xml + localize llap jars
// they cannot be referred to directly as it would be a circular depedency
conf.addResource("llap-daemon-site.xml");
- try {
- final File daemonJar =
- new File(Utilities.jarFinderGetJar(Class
- .forName("org.apache.hadoop.hive.llap.io.api.impl.LlapInputFormat")));
- final LocalResource daemonLr =
- createJarLocalResource(daemonJar.toURI().toURL().toExternalForm());
- commonLocalResources.put(utils.getBaseName(daemonLr), daemonLr);
- } catch (ClassNotFoundException ce) {
- throw new IOException("Cannot find LlapInputFormat in the classpath", ce);
- }
- try {
- final File registryJar =
- new File(Utilities.jarFinderGetJar(Class
- .forName("org.apache.hadoop.registry.client.api.RegistryOperations")));
- final LocalResource registryLr =
- createJarLocalResource(registryJar.toURI().toURL().toExternalForm());
- commonLocalResources.put(utils.getBaseName(registryLr), registryLr);
- } catch (ClassNotFoundException ce) {
- throw new IOException("Cannot find Hadoop Registry in the classpath", ce);
- }
+ addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
+ addJarLRByClass(LlapProtocolClientImpl.class, commonLocalResources);
+ addJarLRByClass(LlapProtocolClientProxy.class, commonLocalResources);
+ addJarLRByClassName("org.apache.hadoop.registry.client.api.RegistryOperations", commonLocalResources);
}
// Create environment for AM.
@@ -587,6 +573,26 @@ public class TezSessionState {
return utils.localizeResource(localFile, destFile, LocalResourceType.FILE, conf);
}
+ private void addJarLRByClassName(String className, final Map<String, LocalResource> lrMap) throws
+ IOException, LoginException {
+ Class<?> clazz;
+ try {
+ clazz = Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ throw new IOException("Cannot find " + className + " in classpath", e);
+ }
+ addJarLRByClass(clazz, lrMap);
+ }
+
+ private void addJarLRByClass(Class<?> clazz, final Map<String, LocalResource> lrMap) throws IOException,
+ LoginException {
+ final File jar =
+ new File(Utilities.jarFinderGetJar(clazz));
+ final LocalResource jarLr =
+ createJarLocalResource(jar.toURI().toURL().toExternalForm());
+ lrMap.put(utils.getBaseName(jarLr), jarLr);
+ }
+
private String getSha(Path localFile) throws IOException, IllegalArgumentException {
InputStream is = null;
try {
[08/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
new file mode 100644
index 0000000..515bf3c
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+public class LlapNodeId {
+
+ private static final LoadingCache<LlapNodeId, LlapNodeId> CACHE =
+ CacheBuilder.newBuilder().softValues().build(
+ new CacheLoader<LlapNodeId, LlapNodeId>() {
+ @Override
+ public LlapNodeId load(LlapNodeId key) throws Exception {
+ return key;
+ }
+ });
+
+ public static LlapNodeId getInstance(String hostname, int port) {
+ return CACHE.getUnchecked(new LlapNodeId(hostname, port));
+ }
+
+
+ private final String hostname;
+ private final int port;
+
+
+ private LlapNodeId(String hostname, int port) {
+ this.hostname = hostname;
+ this.port = port;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ LlapNodeId that = (LlapNodeId) o;
+
+ if (port != that.port) {
+ return false;
+ }
+ if (!hostname.equals(that.hostname)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = hostname.hashCode();
+ result = 1009 * result + port;
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return hostname + ":" + port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java
new file mode 100644
index 0000000..cd11bdb
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapManagementProtocolClientImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.impl;
+
+import javax.annotation.Nullable;
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
+import org.apache.hadoop.security.UserGroupInformation;
+
+public class LlapManagementProtocolClientImpl implements LlapManagementProtocolPB {
+
+ private final Configuration conf;
+ private final InetSocketAddress serverAddr;
+ private final RetryPolicy retryPolicy;
+ private final SocketFactory socketFactory;
+ LlapManagementProtocolPB proxy;
+
+
+ public LlapManagementProtocolClientImpl(Configuration conf, String hostname, int port,
+ @Nullable RetryPolicy retryPolicy,
+ @Nullable SocketFactory socketFactory) {
+ this.conf = conf;
+ this.serverAddr = NetUtils.createSocketAddr(hostname, port);
+ this.retryPolicy = retryPolicy;
+ if (socketFactory == null) {
+ this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+ } else {
+ this.socketFactory = socketFactory;
+ }
+ }
+
+ public LlapManagementProtocolPB getProxy() throws IOException {
+ if (proxy == null) {
+ proxy = createProxy();
+ }
+ return proxy;
+ }
+
+ public LlapManagementProtocolPB createProxy() throws IOException {
+ RPC.setProtocolEngine(conf, LlapManagementProtocolPB.class, ProtobufRpcEngine.class);
+ ProtocolProxy<LlapManagementProtocolPB> proxy =
+ RPC.getProtocolProxy(LlapManagementProtocolPB.class, 0, serverAddr,
+ UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
+ retryPolicy);
+ return proxy.getProxy();
+ }
+
+ @Override
+ public GetTokenResponseProto getDelegationToken(RpcController controller,
+ GetTokenRequestProto request) throws ServiceException {
+ try {
+ return getProxy().getDelegationToken(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java
new file mode 100644
index 0000000..9234b7d
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/impl/LlapProtocolClientImpl.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.impl;
+
+import javax.annotation.Nullable;
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.security.UserGroupInformation;
+
+// TODO Change all this to be based on a regular interface instead of relying on the Proto service - Exception signatures cannot be controlled without this for the moment.
+
+
+public class LlapProtocolClientImpl implements LlapProtocolBlockingPB {
+
+ private final Configuration conf;
+ private final InetSocketAddress serverAddr;
+ private final RetryPolicy retryPolicy;
+ private final SocketFactory socketFactory;
+ LlapProtocolBlockingPB proxy;
+
+
+ public LlapProtocolClientImpl(Configuration conf, String hostname, int port,
+ @Nullable RetryPolicy retryPolicy,
+ @Nullable SocketFactory socketFactory) {
+ this.conf = conf;
+ this.serverAddr = NetUtils.createSocketAddr(hostname, port);
+ this.retryPolicy = retryPolicy;
+ if (socketFactory == null) {
+ this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+ } else {
+ this.socketFactory = socketFactory;
+ }
+ }
+
+ @Override
+ public SubmitWorkResponseProto submitWork(RpcController controller,
+ SubmitWorkRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().submitWork(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
+ SourceStateUpdatedRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().sourceStateUpdated(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public QueryCompleteResponseProto queryComplete(RpcController controller,
+ QueryCompleteRequestProto request) throws
+ ServiceException {
+ try {
+ return getProxy().queryComplete(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public TerminateFragmentResponseProto terminateFragment(
+ RpcController controller,
+ TerminateFragmentRequestProto request) throws ServiceException {
+ try {
+ return getProxy().terminateFragment(null, request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ public LlapProtocolBlockingPB getProxy() throws IOException {
+ if (proxy == null) {
+ proxy = createProxy();
+ }
+ return proxy;
+ }
+
+ public LlapProtocolBlockingPB createProxy() throws IOException {
+ RPC.setProtocolEngine(conf, LlapProtocolBlockingPB.class, ProtobufRpcEngine.class);
+ ProtocolProxy<LlapProtocolBlockingPB> proxy =
+ RPC.getProtocolProxy(LlapProtocolBlockingPB.class, 0, serverAddr,
+ UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
+ retryPolicy);
+ return proxy.getProxy();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java
new file mode 100644
index 0000000..ff215d4
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapManagementProtocolPB.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.llap.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+
+@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB", protocolVersion = 1)
+@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
+@InterfaceAudience.Private
+public interface LlapManagementProtocolPB extends LlapDaemonProtocolProtos.LlapManagementProtocol.BlockingInterface {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java
new file mode 100644
index 0000000..2bd56ca
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapProtocolBlockingPB.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.security.LlapTokenSelector;
+
+@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB", protocolVersion = 1)
+@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
+@TokenInfo(LlapTokenSelector.class)
+@InterfaceAudience.Private
+public interface LlapProtocolBlockingPB extends LlapDaemonProtocolProtos.LlapDaemonProtocol.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
new file mode 100644
index 0000000..9549567
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.protocol;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+import org.apache.tez.runtime.common.security.JobTokenSelector;
+
+@TokenInfo(JobTokenSelector.class)
+public interface LlapTaskUmbilicalProtocol extends VersionedProtocol {
+
+ public static final long versionID = 1L;
+
+ // From Tez. Eventually changes over to the LLAP protocol and ProtocolBuffers
+ boolean canCommit(TezTaskAttemptID taskid) throws IOException;
+ public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
+ throws IOException, TezException;
+
+ public void nodeHeartbeat(Text hostname, int port) throws IOException;
+
+ public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
new file mode 100644
index 0000000..f0bb495
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.security;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+
+/** For now, a LLAP token gives access to any LLAP server. */
+public class LlapTokenIdentifier extends AbstractDelegationTokenIdentifier {
+ private static final String KIND = "LLAP_TOKEN";
+ public static final Text KIND_NAME = new Text(KIND);
+
+ public LlapTokenIdentifier() {
+ super();
+ }
+
+ public LlapTokenIdentifier(Text owner, Text renewer, Text realUser) {
+ super(owner, renewer, realUser);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ // Nothing right now.
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ // Nothing right now.
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public int hashCode() {
+ return -1;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (other != null) && other.getClass().isAssignableFrom(this.getClass());
+ }
+
+ @Override
+ public String toString() {
+ return KIND;
+ }
+
+ @InterfaceAudience.Private
+ public static class Renewer extends Token.TrivialRenewer {
+ @Override
+ protected Text getKind() {
+ return KIND_NAME;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
new file mode 100644
index 0000000..2e99a28
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.security;
+
+import java.io.IOException;
+
+import org.apache.hadoop.security.token.Token;
+
+public interface LlapTokenProvider {
+ Token<LlapTokenIdentifier> getDelegationToken() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
new file mode 100644
index 0000000..b6e7499
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.security;
+
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+
+public class LlapTokenSelector implements TokenSelector<LlapTokenIdentifier> {
+ private static final Log LOG = LogFactory.getLog(LlapTokenSelector.class);
+
+ @Override
+ public Token<LlapTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) return null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking for a token with service " + service);
+ }
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Token = " + token.getKind() + "; service = " + token.getService());
+ }
+ if (LlapTokenIdentifier.KIND_NAME.equals(token.getKind())
+ && service.equals(token.getService())) {
+ @SuppressWarnings("unchecked")
+ Token<LlapTokenIdentifier> result = (Token<LlapTokenIdentifier>)token;
+ return result;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
new file mode 100644
index 0000000..a5c3631
--- /dev/null
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/tez/Converters.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GroupInputSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto;
+import org.apache.tez.common.TezCommonUtils;
+import org.apache.tez.dag.api.EntityDescriptor;
+import org.apache.tez.dag.api.InputDescriptor;
+import org.apache.tez.dag.api.OutputDescriptor;
+import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.GroupInputSpec;
+import org.apache.tez.runtime.api.impl.InputSpec;
+import org.apache.tez.runtime.api.impl.OutputSpec;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+
+public class Converters {
+
+ public static TaskSpec getTaskSpecfromProto(FragmentSpecProto FragmentSpecProto) {
+ TezTaskAttemptID taskAttemptID =
+ TezTaskAttemptID.fromString(FragmentSpecProto.getFragmentIdentifierString());
+
+ ProcessorDescriptor processorDescriptor = null;
+ if (FragmentSpecProto.hasProcessorDescriptor()) {
+ processorDescriptor = convertProcessorDescriptorFromProto(
+ FragmentSpecProto.getProcessorDescriptor());
+ }
+
+ List<InputSpec> inputSpecList = new ArrayList<InputSpec>(FragmentSpecProto.getInputSpecsCount());
+ if (FragmentSpecProto.getInputSpecsCount() > 0) {
+ for (IOSpecProto inputSpecProto : FragmentSpecProto.getInputSpecsList()) {
+ inputSpecList.add(getInputSpecFromProto(inputSpecProto));
+ }
+ }
+
+ List<OutputSpec> outputSpecList =
+ new ArrayList<OutputSpec>(FragmentSpecProto.getOutputSpecsCount());
+ if (FragmentSpecProto.getOutputSpecsCount() > 0) {
+ for (IOSpecProto outputSpecProto : FragmentSpecProto.getOutputSpecsList()) {
+ outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
+ }
+ }
+
+ List<GroupInputSpec> groupInputSpecs =
+ new ArrayList<GroupInputSpec>(FragmentSpecProto.getGroupedInputSpecsCount());
+ if (FragmentSpecProto.getGroupedInputSpecsCount() > 0) {
+ for (GroupInputSpecProto groupInputSpecProto : FragmentSpecProto.getGroupedInputSpecsList()) {
+ groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
+ }
+ }
+
+ TaskSpec taskSpec =
+ new TaskSpec(taskAttemptID, FragmentSpecProto.getDagName(), FragmentSpecProto.getVertexName(),
+ FragmentSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList,
+ outputSpecList, groupInputSpecs);
+ return taskSpec;
+ }
+
+ public static FragmentSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
+ FragmentSpecProto.Builder builder = FragmentSpecProto.newBuilder();
+ builder.setFragmentIdentifierString(taskSpec.getTaskAttemptID().toString());
+ builder.setDagName(taskSpec.getDAGName());
+ builder.setVertexName(taskSpec.getVertexName());
+ builder.setVertexParallelism(taskSpec.getVertexParallelism());
+ builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId());
+ builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId());
+
+ if (taskSpec.getProcessorDescriptor() != null) {
+ builder.setProcessorDescriptor(
+ convertToProto(taskSpec.getProcessorDescriptor()));
+ }
+
+ if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
+ for (InputSpec inputSpec : taskSpec.getInputs()) {
+ builder.addInputSpecs(convertInputSpecToProto(inputSpec));
+ }
+ }
+
+ if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
+ for (OutputSpec outputSpec : taskSpec.getOutputs()) {
+ builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
+ }
+ }
+
+ if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
+ for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
+ builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
+
+ }
+ }
+ return builder.build();
+ }
+
+ private static ProcessorDescriptor convertProcessorDescriptorFromProto(
+ EntityDescriptorProto proto) {
+ String className = proto.getClassName();
+ UserPayload payload = convertPayloadFromProto(proto);
+ ProcessorDescriptor pd = ProcessorDescriptor.create(className);
+ setUserPayload(pd, payload);
+ return pd;
+ }
+
+ private static EntityDescriptorProto convertToProto(
+ EntityDescriptor<?> descriptor) {
+ EntityDescriptorProto.Builder builder = EntityDescriptorProto
+ .newBuilder();
+ builder.setClassName(descriptor.getClassName());
+
+ UserPayload userPayload = descriptor.getUserPayload();
+ if (userPayload != null) {
+ UserPayloadProto.Builder payloadBuilder = UserPayloadProto.newBuilder();
+ if (userPayload.hasPayload()) {
+ payloadBuilder.setUserPayload(ByteString.copyFrom(userPayload.getPayload()));
+ payloadBuilder.setVersion(userPayload.getVersion());
+ }
+ builder.setUserPayload(payloadBuilder.build());
+ }
+ if (descriptor.getHistoryText() != null) {
+ try {
+ builder.setHistoryText(TezCommonUtils.compressByteArrayToByteString(
+ descriptor.getHistoryText().getBytes("UTF-8")));
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+ return builder.build();
+ }
+
+ private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
+ InputDescriptor inputDescriptor = null;
+ if (inputSpecProto.hasIoDescriptor()) {
+ inputDescriptor =
+ convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor());
+ }
+ InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
+ inputSpecProto.getPhysicalEdgeCount());
+ return inputSpec;
+ }
+
+ private static InputDescriptor convertInputDescriptorFromProto(
+ EntityDescriptorProto proto) {
+ String className = proto.getClassName();
+ UserPayload payload = convertPayloadFromProto(proto);
+ InputDescriptor id = InputDescriptor.create(className);
+ setUserPayload(id, payload);
+ return id;
+ }
+
+ private static OutputDescriptor convertOutputDescriptorFromProto(
+ EntityDescriptorProto proto) {
+ String className = proto.getClassName();
+ UserPayload payload = convertPayloadFromProto(proto);
+ OutputDescriptor od = OutputDescriptor.create(className);
+ setUserPayload(od, payload);
+ return od;
+ }
+
+ private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
+ IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+ if (inputSpec.getSourceVertexName() != null) {
+ builder.setConnectedVertexName(inputSpec.getSourceVertexName());
+ }
+ if (inputSpec.getInputDescriptor() != null) {
+ builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor()));
+ }
+ builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
+ return builder.build();
+ }
+
+ private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
+ OutputDescriptor outputDescriptor = null;
+ if (outputSpecProto.hasIoDescriptor()) {
+ outputDescriptor =
+ convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor());
+ }
+ OutputSpec outputSpec =
+ new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
+ outputSpecProto.getPhysicalEdgeCount());
+ return outputSpec;
+ }
+
+ public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
+ IOSpecProto.Builder builder = IOSpecProto.newBuilder();
+ if (outputSpec.getDestinationVertexName() != null) {
+ builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
+ }
+ if (outputSpec.getOutputDescriptor() != null) {
+ builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor()));
+ }
+ builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
+ return builder.build();
+ }
+
+ private static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
+ GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
+ groupInputSpecProto.getGroupVerticesList(),
+ convertInputDescriptorFromProto(groupInputSpecProto.getMergedInputDescriptor()));
+ return groupSpec;
+ }
+
+ private static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
+ GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
+ builder.setGroupName(groupInputSpec.getGroupName());
+ builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
+ builder.setMergedInputDescriptor(convertToProto(groupInputSpec.getMergedInputDescriptor()));
+ return builder.build();
+ }
+
+
+ private static void setUserPayload(EntityDescriptor<?> entity, UserPayload payload) {
+ if (payload != null) {
+ entity.setUserPayload(payload);
+ }
+ }
+
+ private static UserPayload convertPayloadFromProto(
+ EntityDescriptorProto proto) {
+ UserPayload userPayload = null;
+ if (proto.hasUserPayload()) {
+ if (proto.getUserPayload().hasUserPayload()) {
+ userPayload =
+ UserPayload.create(proto.getUserPayload().getUserPayload().asReadOnlyByteBuffer(), proto.getUserPayload().getVersion());
+ } else {
+ userPayload = UserPayload.create(null);
+ }
+ }
+ return userPayload;
+ }
+
+ public static SourceStateProto fromVertexState(VertexState state) {
+ switch (state) {
+ case SUCCEEDED:
+ return SourceStateProto.S_SUCCEEDED;
+ case RUNNING:
+ return SourceStateProto.S_RUNNING;
+ default:
+ throw new RuntimeException("Unexpected state: " + state);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/protobuf/LlapDaemonProtocol.proto
----------------------------------------------------------------------
diff --git a/llap-common/src/protobuf/LlapDaemonProtocol.proto b/llap-common/src/protobuf/LlapDaemonProtocol.proto
new file mode 100644
index 0000000..944c96c
--- /dev/null
+++ b/llap-common/src/protobuf/LlapDaemonProtocol.proto
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hive.llap.daemon.rpc";
+option java_outer_classname = "LlapDaemonProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+// TODO Change this as the interface evolves. Currently using Tez constructs.
+
+message UserPayloadProto {
+ optional bytes user_payload = 1;
+ optional int32 version = 2;
+}
+
+message EntityDescriptorProto {
+ optional string class_name = 1;
+ optional UserPayloadProto user_payload = 2;
+ optional bytes history_text = 3;
+}
+
+message IOSpecProto {
+ optional string connected_vertex_name = 1;
+ optional EntityDescriptorProto io_descriptor = 2;
+ optional int32 physical_edge_count = 3;
+}
+
+message GroupInputSpecProto {
+ optional string group_name = 1;
+ repeated string group_vertices = 2;
+ optional EntityDescriptorProto merged_input_descriptor = 3;
+}
+
+
+message FragmentSpecProto {
+ optional string fragment_identifier_string = 1;
+ optional string dag_name = 2;
+ optional int32 dag_id = 11;
+ optional string vertex_name = 3;
+ optional EntityDescriptorProto processor_descriptor = 4;
+ repeated IOSpecProto input_specs = 5;
+ repeated IOSpecProto output_specs = 6;
+ repeated GroupInputSpecProto grouped_input_specs = 7;
+ optional int32 vertex_parallelism = 8;
+ optional int32 fragment_number =9;
+ optional int32 attempt_number = 10;
+}
+
+message FragmentRuntimeInfo {
+ optional int32 num_self_and_upstream_tasks = 1;
+ optional int32 num_self_and_upstream_completed_tasks = 2;
+ optional int32 within_dag_priority = 3;
+ optional int64 dag_start_time = 4;
+ optional int64 first_attempt_start_time = 5;
+ optional int64 current_attempt_start_time = 6;
+}
+
+enum SourceStateProto {
+ S_SUCCEEDED = 1;
+ S_RUNNING = 2;
+}
+
+message QueryIdentifierProto {
+ optional string app_identifier = 1;
+ optional int32 dag_identifier = 2;
+}
+
+message SubmitWorkRequestProto {
+ optional string container_id_string = 1;
+ optional string am_host = 2;
+ optional int32 am_port = 3;
+ optional string token_identifier = 4;
+ optional bytes credentials_binary = 5;
+ optional string user = 6;
+ optional string application_id_string = 7;
+ optional int32 app_attempt_number = 8;
+ optional FragmentSpecProto fragment_spec = 9;
+ optional FragmentRuntimeInfo fragment_runtime_info = 10;
+}
+
+enum SubmissionStateProto {
+ ACCEPTED = 1;
+ REJECTED = 2;
+ EVICTED_OTHER = 3;
+}
+
+message SubmitWorkResponseProto {
+ optional SubmissionStateProto submission_state = 1;
+}
+
+message SourceStateUpdatedRequestProto {
+ optional QueryIdentifierProto query_identifier = 1;
+ optional string src_name = 2;
+ optional SourceStateProto state = 3;
+}
+
+message SourceStateUpdatedResponseProto {
+}
+
+message QueryCompleteRequestProto {
+ optional string query_id = 1;
+ optional QueryIdentifierProto query_identifier = 2;
+ optional int64 delete_delay = 4 [default = 0];
+}
+
+message QueryCompleteResponseProto {
+}
+
+message TerminateFragmentRequestProto {
+ optional QueryIdentifierProto query_identifier = 1;
+ optional string fragment_identifier_string = 2;
+}
+
+message TerminateFragmentResponseProto {
+}
+
+message GetTokenRequestProto {
+}
+
+message GetTokenResponseProto {
+ optional bytes token = 1;
+}
+
+service LlapDaemonProtocol {
+ rpc submitWork(SubmitWorkRequestProto) returns (SubmitWorkResponseProto);
+ rpc sourceStateUpdated(SourceStateUpdatedRequestProto) returns (SourceStateUpdatedResponseProto);
+ rpc queryComplete(QueryCompleteRequestProto) returns (QueryCompleteResponseProto);
+ rpc terminateFragment(TerminateFragmentRequestProto) returns (TerminateFragmentResponseProto);
+}
+
+service LlapManagementProtocol {
+ rpc getDelegationToken(GetTokenRequestProto) returns (GetTokenResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java
----------------------------------------------------------------------
diff --git a/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java b/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java
new file mode 100644
index 0000000..fec1c35
--- /dev/null
+++ b/llap-common/src/test/org/apache/hadoop/hive/llap/testhelpers/ControlledClock.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.llap.testhelpers;
+
+import org.apache.hadoop.yarn.util.Clock;
+
+public class ControlledClock implements Clock {
+ private long time = -1;
+ private final Clock actualClock;
+ public ControlledClock(Clock actualClock) {
+ this.actualClock = actualClock;
+ }
+ public synchronized void setTime(long time) {
+ this.time = time;
+ }
+ public synchronized void reset() {
+ time = -1;
+ }
+
+ @Override
+ public synchronized long getTime() {
+ if (time != -1) {
+ return time;
+ }
+ return actualClock.getTime();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index ab25f66..4011da0 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -32,7 +32,7 @@
</properties>
<dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+ <!-- dependencies are always listed in sorted order by groupId, artifactId -->
<!-- intra-project -->
<dependency>
<groupId>org.apache.hive</groupId>
@@ -46,11 +46,21 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-llap-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-tez</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -90,77 +100,66 @@
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-runtime-internals</artifactId>
- <version>${tez.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.tez</groupId>
- <artifactId>tez-runtime-library</artifactId>
- <version>${tez.version}</version>
- <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-mapreduce</artifactId>
+ <artifactId>tez-runtime-internals</artifactId>
<version>${tez.version}</version>
<optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-dag</artifactId>
+ <artifactId>tez-runtime-library</artifactId>
<version>${tez.version}</version>
<optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+
<!-- test inter-project -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -168,17 +167,17 @@
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
@@ -225,15 +224,15 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -248,54 +247,6 @@
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>protobuf</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>generate-protobuf-sources</id>
- <phase>generate-sources</phase>
- <configuration>
- <target>
- <property name="protobuf.src.dir" location="${basedir}/src/protobuf"/>
- <property name="protobuf.build.dir" location="${basedir}/src/gen/protobuf/gen-java"/>
- <echo>Building LLAP-Server Protobuf</echo>
- <mkdir dir="${protobuf.build.dir}"/>
- <exec executable="protoc" failonerror="true">
- <arg value="--java_out=${protobuf.build.dir}"/>
- <arg value="-I=${protobuf.src.dir}"/>
- <arg value="${protobuf.src.dir}/LlapDaemonProtocol.proto"/>
- </exec>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
-
<build>
<sourceDirectory>${basedir}/src/java</sourceDirectory>
<testSourceDirectory>${basedir}/src/test</testSourceDirectory>
[02/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
new file mode 100644
index 0000000..3bca0da
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
@@ -0,0 +1,1512 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.commons.lang.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.registry.ServiceInstance;
+import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
+import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Clock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskScheduler;
+import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapTaskSchedulerService extends TaskScheduler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LlapTaskSchedulerService.class);
+
+ private final Configuration conf;
+
+ // interface into the registry service
+ private ServiceInstanceSet activeInstances;
+
+ // Tracks all instances, including ones which have been disabled in the past.
+ // LinkedHashMap to provide the same iteration order when selecting a random host.
+ @VisibleForTesting
+ final Map<ServiceInstance, NodeInfo> instanceToNodeMap = new LinkedHashMap<>();
+ // TODO Ideally, remove elements from this once it's known that no tasks are linked to the instance (all deallocated)
+
+ // Tracks tasks which could not be allocated immediately.
+ @VisibleForTesting
+ final TreeMap<Priority, List<TaskInfo>> pendingTasks = new TreeMap<>(new Comparator<Priority>() {
+ @Override
+ public int compare(Priority o1, Priority o2) {
+ return o1.getPriority() - o2.getPriority();
+ }
+ });
+
+ // Tracks running and queued tasks. Cleared after a task completes.
+ private final ConcurrentMap<Object, TaskInfo> knownTasks = new ConcurrentHashMap<>();
+ private final TreeMap<Integer, TreeSet<TaskInfo>> runningTasks = new TreeMap<>();
+ private static final TaskStartComparator TASK_INFO_COMPARATOR = new TaskStartComparator();
+
+ // Queue for disabled nodes. Nodes make it out of this queue when their expiration timeout is hit.
+ @VisibleForTesting
+ final DelayQueue<NodeInfo> disabledNodesQueue = new DelayQueue<>();
+
+ private final boolean forceLocation;
+
+ private final ContainerFactory containerFactory;
+ private final Random random = new Random();
+ private final Clock clock;
+
+ private final ListeningExecutorService nodeEnabledExecutor;
+ private final NodeEnablerCallable nodeEnablerCallable =
+ new NodeEnablerCallable();
+
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
+ private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
+
+ private final Lock scheduleLock = new ReentrantLock();
+ private final Condition scheduleCondition = scheduleLock.newCondition();
+ private final AtomicBoolean pendingScheduleInvodations = new AtomicBoolean(false);
+ private final ListeningExecutorService schedulerExecutor;
+ private final SchedulerCallable schedulerCallable = new SchedulerCallable();
+
+ private final AtomicBoolean isStopped = new AtomicBoolean(false);
+ // Tracks total pending preemptions.
+ private final AtomicInteger pendingPreemptions = new AtomicInteger(0);
+ // Tracks pending preemptions per host, using the hostname || Always to be accessed inside a lock
+ private final Map<String, MutableInt> pendingPreemptionsPerHost = new HashMap<>();
+
+ private final NodeBlacklistConf nodeBlacklistConf;
+
+ // Per daemon
+ private final int memoryPerInstance;
+ private final int coresPerInstance;
+ private final int executorsPerInstance;
+
+ private final int numSchedulableTasksPerNode;
+
+ // Per Executor Thread
+ private final Resource resourcePerExecutor;
+
+ private final LlapRegistryService registry = new LlapRegistryService(false);
+
+ private volatile ListenableFuture<Void> nodeEnablerFuture;
+ private volatile ListenableFuture<Void> schedulerFuture;
+
+ @VisibleForTesting
+ private final AtomicInteger dagCounter = new AtomicInteger(1);
+ // Statistics to track allocations
+ // All of stats variables are visible for testing.
+ @VisibleForTesting
+ StatsPerDag dagStats = new StatsPerDag();
+
+ public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) {
+ this(taskSchedulerContext, new SystemClock());
+ }
+
+ @VisibleForTesting
+ public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock) {
+ super(taskSchedulerContext);
+ this.clock = clock;
+ try {
+ this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
+ } catch (IOException e) {
+ throw new TezUncheckedException(
+ "Failed to parse user payload for " + LlapTaskSchedulerService.class.getSimpleName(), e);
+ }
+ this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(),
+ taskSchedulerContext.getCustomClusterIdentifier());
+ this.memoryPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB);
+ this.coresPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE);
+ this.executorsPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
+ this.nodeBlacklistConf = new NodeBlacklistConf(
+ HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS,
+ TimeUnit.MILLISECONDS),
+ HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MAX_TIMEOUT_MS,
+ TimeUnit.MILLISECONDS),
+ HiveConf.getFloatVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_DISABLE_BACK_OFF_FACTOR));
+
+ this.numSchedulableTasksPerNode =
+ HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NUM_SCHEDULABLE_TASKS_PER_NODE);
+
+ long localityDelayMs = HiveConf
+ .getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY, TimeUnit.MILLISECONDS);
+ if (localityDelayMs == -1) {
+ this.forceLocation = true;
+ } else {
+ this.forceLocation = false;
+ }
+
+ int memoryPerExecutor = (int) (memoryPerInstance / (float) executorsPerInstance);
+ int coresPerExecutor = (int) (coresPerInstance / (float) executorsPerInstance);
+ this.resourcePerExecutor = Resource.newInstance(memoryPerExecutor, coresPerExecutor);
+
+ String instanceId = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
+
+ Preconditions.checkNotNull(instanceId, ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname
+ + " must be defined");
+
+ ExecutorService executorServiceRaw =
+ Executors.newFixedThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapSchedulerNodeEnabler").build());
+ nodeEnabledExecutor = MoreExecutors.listeningDecorator(executorServiceRaw);
+
+ ExecutorService schedulerExecutorServiceRaw = Executors.newFixedThreadPool(1,
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build());
+ schedulerExecutor = MoreExecutors.listeningDecorator(schedulerExecutorServiceRaw);
+
+ LOG.info("Running with configuration: " + "memoryPerInstance=" + memoryPerInstance
+ + ", vCoresPerInstance=" + coresPerInstance + ", executorsPerInstance="
+ + executorsPerInstance + ", resourcePerInstanceInferred=" + resourcePerExecutor
+ + ", nodeBlacklistConf=" + nodeBlacklistConf
+ + ", forceLocation=" + forceLocation);
+ }
+
+ @Override
+ public void initialize() {
+ registry.init(conf);
+ }
+
+ @Override
+ public void start() throws IOException {
+ writeLock.lock();
+ try {
+ nodeEnablerFuture = nodeEnabledExecutor.submit(nodeEnablerCallable);
+ Futures.addCallback(nodeEnablerFuture, new FutureCallback<Void>() {
+ @Override
+ public void onSuccess(Void result) {
+ LOG.info("NodeEnabledThread exited");
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.warn("NodeEnabledThread exited with error", t);
+ }
+ });
+ schedulerFuture = schedulerExecutor.submit(schedulerCallable);
+ Futures.addCallback(schedulerFuture, new FutureCallback<Void>() {
+ @Override
+ public void onSuccess(Void result) {
+ LOG.info("SchedulerThread exited");
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.warn("SchedulerThread exited with error", t);
+ }
+ });
+ registry.start();
+ activeInstances = registry.getInstances();
+ for (ServiceInstance inst : activeInstances.getAll().values()) {
+ addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ writeLock.lock();
+ try {
+ if (!this.isStopped.getAndSet(true)) {
+ nodeEnablerCallable.shutdown();
+ if (nodeEnablerFuture != null) {
+ nodeEnablerFuture.cancel(true);
+ }
+ nodeEnabledExecutor.shutdownNow();
+
+ schedulerCallable.shutdown();
+ if (schedulerFuture != null) {
+ schedulerFuture.cancel(true);
+ }
+ schedulerExecutor.shutdownNow();
+
+ if (registry != null) {
+ registry.stop();
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Resource getTotalResources() {
+ int memory = 0;
+ int vcores = 0;
+ readLock.lock();
+ try {
+ for (ServiceInstance inst : activeInstances.getAll().values()) {
+ if (inst.isAlive()) {
+ Resource r = inst.getResource();
+ LOG.info("Found instance " + inst);
+ memory += r.getMemory();
+ vcores += r.getVirtualCores();
+ } else {
+ LOG.info("Ignoring dead instance " + inst);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ return Resource.newInstance(memory, vcores);
+ }
+
+ /**
+ * The difference between this and getTotalResources() is that this only gives currently free
+ * resource instances, while the other lists all the instances that may become available in a
+ * while.
+ */
+ @Override
+ public Resource getAvailableResources() {
+ // need a state store eventually for current state & measure backoffs
+ int memory = 0;
+ int vcores = 0;
+ readLock.lock();
+ try {
+ for (Entry<ServiceInstance, NodeInfo> entry : instanceToNodeMap.entrySet()) {
+ if (entry.getKey().isAlive() && !entry.getValue().isDisabled()) {
+ Resource r = entry.getKey().getResource();
+ memory += r.getMemory();
+ vcores += r.getVirtualCores();
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ return Resource.newInstance(memory, vcores);
+ }
+
+ @Override
+ public int getClusterNodeCount() {
+ readLock.lock();
+ try {
+ int n = 0;
+ for (ServiceInstance inst : activeInstances.getAll().values()) {
+ if (inst.isAlive()) {
+ n++;
+ }
+ }
+ return n;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void dagComplete() {
+ // This is effectively DAG completed, and can be used to reset statistics being tracked.
+ LOG.info("DAG: " + dagCounter.get() + " completed. Scheduling stats: " + dagStats);
+ dagCounter.incrementAndGet();
+ dagStats = new StatsPerDag();
+ }
+
+ @Override
+ public void blacklistNode(NodeId nodeId) {
+ LOG.info("BlacklistNode not supported");
+ }
+
+ @Override
+ public void unblacklistNode(NodeId nodeId) {
+ LOG.info("unBlacklistNode not supported");
+ }
+
+ @Override
+ public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
+ Priority priority, Object containerSignature, Object clientCookie) {
+ TaskInfo taskInfo =
+ new TaskInfo(task, clientCookie, priority, capability, hosts, racks, clock.getTime());
+ writeLock.lock();
+ try {
+ dagStats.registerTaskRequest(hosts, racks);
+ } finally {
+ writeLock.unlock();
+ }
+ addPendingTask(taskInfo);
+ trySchedulingPendingTasks();
+ }
+
+ @Override
+ public void allocateTask(Object task, Resource capability, ContainerId containerId,
+ Priority priority, Object containerSignature, Object clientCookie) {
+ // Container affinity can be implemented as Host affinity for LLAP. Not required until
+ // 1:1 edges are used in Hive.
+ TaskInfo taskInfo =
+ new TaskInfo(task, clientCookie, priority, capability, null, null, clock.getTime());
+ writeLock.lock();
+ try {
+ dagStats.registerTaskRequest(null, null);
+ } finally {
+ writeLock.unlock();
+ }
+ addPendingTask(taskInfo);
+ trySchedulingPendingTasks();
+ }
+
+
+ // This may be invoked before a container is ever assigned to a task. allocateTask... app decides
+ // the task is no longer required, and asks for a de-allocation.
+ @Override
+ public boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason, String diagnostics) {
+ writeLock.lock(); // Updating several local structures
+ TaskInfo taskInfo;
+ try {
+ taskInfo = unregisterTask(task);
+ if (taskInfo == null) {
+ LOG.error("Could not determine ContainerId for task: "
+ + task
+ + " . Could have hit a race condition. Ignoring."
+ + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
+ return false;
+ }
+ if (taskInfo.containerId == null) {
+ if (taskInfo.assigned) {
+ LOG.error("Task: "
+ + task
+ + " assigned, but could not find the corresponding containerId."
+ + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
+ } else {
+ LOG.info("Ignoring deallocate request for task " + task
+ + " which hasn't been assigned to a container");
+ removePendingTask(taskInfo);
+ }
+ return false;
+ }
+ ServiceInstance assignedInstance = taskInfo.assignedInstance;
+ assert assignedInstance != null;
+
+ NodeInfo nodeInfo = instanceToNodeMap.get(assignedInstance);
+ assert nodeInfo != null;
+
+ // Re-enable the node if preempted
+ if (taskInfo.preempted) {
+ LOG.info("Processing deallocateTask for {} which was preempted, EndReason={}", task, endReason);
+ unregisterPendingPreemption(taskInfo.assignedInstance.getHost());
+ nodeInfo.registerUnsuccessfulTaskEnd(true);
+ if (nodeInfo.isDisabled()) {
+ // Re-enable the node. If a task succeeded, a slot may have become available.
+ // Also reset commFailures since a task was able to communicate back and indicate success.
+ nodeInfo.enableNode();
+ // Re-insert into the queue to force the poll thread to remove the element.
+ if ( disabledNodesQueue.remove(nodeInfo)) {
+ disabledNodesQueue.add(nodeInfo);
+ }
+ }
+ // In case of success, trigger a scheduling run for pending tasks.
+ trySchedulingPendingTasks();
+ } else {
+ if (taskSucceeded) {
+ // The node may have been blacklisted at this point - which means it may not be in the
+ // activeNodeList.
+
+ nodeInfo.registerTaskSuccess();
+
+ if (nodeInfo.isDisabled()) {
+ // Re-enable the node. If a task succeeded, a slot may have become available.
+ // Also reset commFailures since a task was able to communicate back and indicate success.
+ nodeInfo.enableNode();
+ // Re-insert into the queue to force the poll thread to remove the element.
+ if (disabledNodesQueue.remove(nodeInfo)) {
+ disabledNodesQueue.add(nodeInfo);
+ }
+ }
+ // In case of success, trigger a scheduling run for pending tasks.
+ trySchedulingPendingTasks();
+
+ } else if (!taskSucceeded) {
+ nodeInfo.registerUnsuccessfulTaskEnd(false);
+ if (endReason != null && EnumSet
+ .of(TaskAttemptEndReason.EXECUTOR_BUSY, TaskAttemptEndReason.COMMUNICATION_ERROR)
+ .contains(endReason)) {
+ if (endReason == TaskAttemptEndReason.COMMUNICATION_ERROR) {
+ dagStats.registerCommFailure(taskInfo.assignedInstance.getHost());
+ } else if (endReason == TaskAttemptEndReason.EXECUTOR_BUSY) {
+ dagStats.registerTaskRejected(taskInfo.assignedInstance.getHost());
+ }
+ }
+ boolean commFailure =
+ endReason != null && endReason == TaskAttemptEndReason.COMMUNICATION_ERROR;
+ disableInstance(assignedInstance, commFailure);
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ getContext().containerBeingReleased(taskInfo.containerId);
+ return true;
+ }
+
+ @Override
+ public Object deallocateContainer(ContainerId containerId) {
+ LOG.debug("Ignoring deallocateContainer for containerId: " + containerId);
+ // Containers are not being tracked for re-use.
+ // This is safe to ignore since a deallocate task will come in.
+ return null;
+ }
+
+ @Override
+ public void setShouldUnregister() {
+
+ }
+
+ @Override
+ public boolean hasUnregistered() {
+ // Nothing to do. No registration involved.
+ return true;
+ }
+
+ private ExecutorService createAppCallbackExecutorService() {
+ return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
+ .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
+ }
+
+ /**
+ * @param request the list of preferred hosts. null implies any host
+ * @return
+ */
+ private SelectHostResult selectHost(TaskInfo request) {
+ String[] requestedHosts = request.requestedHosts;
+ readLock.lock(); // Read-lock. Not updating any stats at the moment.
+ try {
+ // Check if any hosts are active.
+ if (getAvailableResources().getMemory() <= 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Refreshing instances since total memory is 0");
+ }
+ refreshInstances();
+ }
+
+ // If there's no memory available, fail
+ if (getTotalResources().getMemory() <= 0) {
+ return SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY;
+ }
+
+ if (requestedHosts != null && requestedHosts.length > 0) {
+ int prefHostCount = -1;
+ boolean requestedHostExists = false;
+ for (String host : requestedHosts) {
+ prefHostCount++;
+ // Pick the first host always. Weak attempt at cache affinity.
+ Set<ServiceInstance> instances = activeInstances.getByHost(host);
+ if (!instances.isEmpty()) {
+ requestedHostExists = true;
+ for (ServiceInstance inst : instances) {
+ NodeInfo nodeInfo = instanceToNodeMap.get(inst);
+ if (nodeInfo != null && nodeInfo.canAcceptTask()) {
+ LOG.info("Assigning " + inst + " when looking for " + host + "." +
+ " FirstRequestedHost=" + (prefHostCount == 0) +
+ (requestedHosts.length > 1 ? "#prefLocations=" + requestedHosts.length : ""));
+ return new SelectHostResult(inst, nodeInfo);
+ }
+ }
+ }
+ }
+ // Check if forcing the location is required.
+ if (forceLocation) {
+ if (requestedHostExists) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping non-local location allocation for [" + request.task +
+ "] when trying to allocate on [" + Arrays.toString(requestedHosts) + "]");
+ }
+ return SELECT_HOST_RESULT_DELAYED_LOCALITY;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not skipping non-local location allocation for [" + request.task +
+ "] when trying to allocate on [" + Arrays.toString(requestedHosts) +
+ "] since none of these hosts are part of the known list");
+ }
+ }
+ }
+ }
+ /* fall through - miss in locality (random scheduling) */
+ Entry<ServiceInstance, NodeInfo>[] all =
+ instanceToNodeMap.entrySet().toArray(new Entry[instanceToNodeMap.size()]);
+ // Check again
+ if (all.length > 0) {
+ int n = random.nextInt(all.length);
+ // start at random offset and iterate whole list
+ for (int i = 0; i < all.length; i++) {
+ Entry<ServiceInstance, NodeInfo> inst = all[(i + n) % all.length];
+ if (inst.getValue().canAcceptTask()) {
+ LOG.info("Assigning " + inst + " when looking for any host, from #hosts=" + all.length +
+ ", requestedHosts=" +
+ ((requestedHosts == null || requestedHosts.length == 0) ? "null" :
+ Arrays.toString(requestedHosts)));
+ return new SelectHostResult(inst.getKey(), inst.getValue());
+ }
+ }
+ }
+ return SELECT_HOST_RESULT_DELAYED_RESOURCES;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ // TODO Each refresh operation should addNodes if they don't already exist.
+ // Even better would be to get notifications from the service impl when a node gets added or removed.
+ // Instead of having to walk through the entire list. The computation of a node getting added or
+ // removed already exists in the DynamicRegistry implementation.
+ private void refreshInstances() {
+ try {
+ activeInstances.refresh(); // handles its own sync
+ } catch (IOException ioe) {
+ LOG.warn("Could not refresh list of active instances", ioe);
+ }
+ }
+
+ private void scanForNodeChanges() {
+ /* check again whether nodes are disabled or just missing */
+ writeLock.lock();
+ try {
+ for (ServiceInstance inst : activeInstances.getAll().values()) {
+ if (inst.isAlive() && instanceToNodeMap.containsKey(inst) == false) {
+ /* that's a good node, not added to the allocations yet */
+ LOG.info("Found a new node: " + inst + ".");
+ addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void addNode(ServiceInstance inst, NodeInfo node) {
+ LOG.info("Adding node: " + inst);
+ instanceToNodeMap.put(inst, node);
+ // Trigger scheduling since a new node became available.
+ trySchedulingPendingTasks();
+ }
+
+ private void reenableDisabledNode(NodeInfo nodeInfo) {
+ writeLock.lock();
+ try {
+ if (nodeInfo.hadCommFailure()) {
+ // If the node being re-enabled was not marked busy previously, then it was disabled due to
+ // some other failure. Refresh the service list to see if it's been removed permanently.
+ refreshInstances();
+ }
+ LOG.info("Attempting to re-enable node: " + nodeInfo.getServiceInstance());
+ if (nodeInfo.getServiceInstance().isAlive()) {
+ nodeInfo.enableNode();
+ } else {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Removing dead node " + nodeInfo);
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void disableInstance(ServiceInstance instance, boolean isCommFailure) {
+ writeLock.lock();
+ try {
+ NodeInfo nodeInfo = instanceToNodeMap.get(instance);
+ if (nodeInfo == null || nodeInfo.isDisabled()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Node: " + instance + " already disabled, or invalid. Not doing anything.");
+ }
+ } else {
+ nodeInfo.disableNode(isCommFailure);
+ // TODO: handle task to container map events in case of hard failures
+ disabledNodesQueue.add(nodeInfo);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void addPendingTask(TaskInfo taskInfo) {
+ writeLock.lock();
+ try {
+ List<TaskInfo> tasksAtPriority = pendingTasks.get(taskInfo.priority);
+ if (tasksAtPriority == null) {
+ tasksAtPriority = new LinkedList<>();
+ pendingTasks.put(taskInfo.priority, tasksAtPriority);
+ }
+ tasksAtPriority.add(taskInfo);
+ knownTasks.putIfAbsent(taskInfo.task, taskInfo);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /* Remove a task from the pending list */
+ private void removePendingTask(TaskInfo taskInfo) {
+ writeLock.lock();
+ try {
+ Priority priority = taskInfo.priority;
+ List<TaskInfo> taskInfoList = pendingTasks.get(priority);
+ if (taskInfoList == null || taskInfoList.isEmpty() || !taskInfoList.remove(taskInfo)) {
+ LOG.warn("Could not find task: " + taskInfo.task + " in pending list, at priority: "
+ + priority);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /* Register a running task into the runningTasks structure */
+ private void registerRunningTask(TaskInfo taskInfo) {
+ writeLock.lock();
+ try {
+ int priority = taskInfo.priority.getPriority();
+ TreeSet<TaskInfo> tasksAtpriority = runningTasks.get(priority);
+ if (tasksAtpriority == null) {
+ tasksAtpriority = new TreeSet<>(TASK_INFO_COMPARATOR);
+ runningTasks.put(priority, tasksAtpriority);
+ }
+ tasksAtpriority.add(taskInfo);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /* Unregister a task from the known and running structures */
+ private TaskInfo unregisterTask(Object task) {
+ writeLock.lock();
+ try {
+ TaskInfo taskInfo = knownTasks.remove(task);
+ if (taskInfo != null) {
+ if (taskInfo.assigned) {
+ // Remove from the running list.
+ int priority = taskInfo.priority.getPriority();
+ Set<TaskInfo> tasksAtPriority = runningTasks.get(priority);
+ Preconditions.checkState(tasksAtPriority != null,
+ "runningTasks should contain an entry if the task was in running state. Caused by task: {}", task);
+ tasksAtPriority.remove(taskInfo);
+ if (tasksAtPriority.isEmpty()) {
+ runningTasks.remove(priority);
+ }
+ }
+ } else {
+ LOG.warn("Could not find TaskInfo for task: {}. Not removing it from the running set", task);
+ }
+ return taskInfo;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private enum ScheduleResult {
+ // Successfully scheduled
+ SCHEDULED,
+
+ // Delayed to find a local match
+ DELAYED_LOCALITY,
+
+ // Delayed due to temporary resource availability
+ DELAYED_RESOURCES,
+
+ // Inadequate total resources - will never succeed / wait for new executors to become available
+ INADEQUATE_TOTAL_RESOURCES,
+ }
+
+ @VisibleForTesting
+ protected void schedulePendingTasks() {
+ writeLock.lock();
+ try {
+ Iterator<Entry<Priority, List<TaskInfo>>> pendingIterator =
+ pendingTasks.entrySet().iterator();
+ while (pendingIterator.hasNext()) {
+ Entry<Priority, List<TaskInfo>> entry = pendingIterator.next();
+ List<TaskInfo> taskListAtPriority = entry.getValue();
+ Iterator<TaskInfo> taskIter = taskListAtPriority.iterator();
+ boolean scheduledAllAtPriority = true;
+ while (taskIter.hasNext()) {
+
+ // TODO Optimization: Add a check to see if there's any capacity available. No point in
+ // walking through all active nodes, if they don't have potential capacity.
+
+ TaskInfo taskInfo = taskIter.next();
+ if (taskInfo.getNumPreviousAssignAttempts() == 1) {
+ dagStats.registerDelayedAllocation();
+ }
+ taskInfo.triedAssigningTask();
+ ScheduleResult scheduleResult = scheduleTask(taskInfo);
+ if (scheduleResult == ScheduleResult.SCHEDULED) {
+ taskIter.remove();
+ } else {
+ // TODO Handle INADEQUATE_TOTAL_RESOURCES eventually - either by throwin an error immediately,
+ // or waiting for some timeout for new executors and then throwing an error
+
+ // Try pre-empting a task so that a higher priority task can take it's place.
+ // Preempt only if there's no pending preemptions to avoid preempting twice for a task.
+ String[] potentialHosts;
+ if (scheduleResult == ScheduleResult.DELAYED_LOCALITY) {
+ // preempt only on specific hosts, if no preemptions already exist on those.
+ potentialHosts = taskInfo.requestedHosts;
+ //Protect against a bad location being requested.
+ if (potentialHosts == null || potentialHosts.length == 0) {
+ potentialHosts = null;
+ }
+ } else {
+ // preempt on any host.
+ potentialHosts = null;
+ }
+
+ if (potentialHosts != null) {
+ // Preempt on specific host
+ boolean shouldPreempt = true;
+ for (String host : potentialHosts) {
+ // Preempt only if there are not pending preemptions on the same host
+ // When the premption registers, the request at the highest priority will be given the slot,
+ // even if the initial request was for some other task.
+ // TODO Maybe register which task the preemption was for, to avoid a bad non-local allocation.
+ MutableInt pendingHostPreemptions = pendingPreemptionsPerHost.get(host);
+ if (pendingHostPreemptions != null && pendingHostPreemptions.intValue() > 0) {
+ shouldPreempt = false;
+ break;
+ }
+ }
+ if (shouldPreempt) {
+ LOG.info("Attempting to preempt for {}, pendingPreemptions={} on hosts={}",
+ taskInfo.task, pendingPreemptions.get(), Arrays.toString(potentialHosts));
+ preemptTasks(entry.getKey().getPriority(), 1, potentialHosts);
+ }
+ } else {
+ // Request for a preemption if there's none pending. If a single preemption is pending,
+ // and this is the next task to be assigned, it will be assigned once that slot becomes available.
+ if (pendingPreemptions.get() == 0) {
+ LOG.info("Attempting to preempt for {}, pendingPreemptions={} on any host",
+ taskInfo.task, pendingPreemptions.get());
+ preemptTasks(entry.getKey().getPriority(), 1, null);
+ }
+ }
+ // Since there was an allocation failure - don't try assigning tasks at the next priority.
+
+ scheduledAllAtPriority = false;
+ // Don't break if this allocation failure was a result of a LOCALITY_DELAY. Others could still be allocated.
+ if (scheduleResult != ScheduleResult.DELAYED_LOCALITY) {
+ break;
+ }
+ } // end of else - i.e. could not allocate
+ } // end of loop over pending tasks
+ if (taskListAtPriority.isEmpty()) {
+ // Remove the entry, if there's nothing left at the specific priority level
+ pendingIterator.remove();
+ }
+ if (!scheduledAllAtPriority) {
+ // Don't attempt scheduling for additional priorities
+ break;
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+
+ private ScheduleResult scheduleTask(TaskInfo taskInfo) {
+ SelectHostResult selectHostResult = selectHost(taskInfo);
+ if (selectHostResult.scheduleResult == ScheduleResult.SCHEDULED) {
+ NodeServiceInstancePair nsPair = selectHostResult.nodeServiceInstancePair;
+ Container container =
+ containerFactory.createContainer(resourcePerExecutor, taskInfo.priority,
+ nsPair.getServiceInstance().getHost(),
+ nsPair.getServiceInstance().getRpcPort());
+ writeLock.lock(); // While updating local structures
+ try {
+ LOG.info("Assigned task {} to container {}", taskInfo, container.getId());
+ dagStats.registerTaskAllocated(taskInfo.requestedHosts, taskInfo.requestedRacks,
+ nsPair.getServiceInstance().getHost());
+ taskInfo.setAssignmentInfo(nsPair.getServiceInstance(), container.getId(), clock.getTime());
+ registerRunningTask(taskInfo);
+ nsPair.getNodeInfo().registerTaskScheduled();
+ } finally {
+ writeLock.unlock();
+ }
+ getContext().taskAllocated(taskInfo.task, taskInfo.clientCookie, container);
+ }
+ return selectHostResult.scheduleResult;
+ }
+
+ // Removes tasks from the runningList and sends out a preempt request to the system.
+ // Subsequent tasks will be scheduled again once the de-allocate request for the preempted
+ // task is processed.
+ private void preemptTasks(int forPriority, int numTasksToPreempt, String []potentialHosts) {
+ Set<String> preemptHosts;
+ if (potentialHosts == null) {
+ preemptHosts = null;
+ } else {
+ preemptHosts = Sets.newHashSet(potentialHosts);
+ }
+ writeLock.lock();
+ List<TaskInfo> preemptedTaskList = null;
+ try {
+ NavigableMap<Integer, TreeSet<TaskInfo>> orderedMap = runningTasks.descendingMap();
+ Iterator<Entry<Integer, TreeSet<TaskInfo>>> iterator = orderedMap.entrySet().iterator();
+ int preemptedCount = 0;
+ while (iterator.hasNext() && preemptedCount < numTasksToPreempt) {
+ Entry<Integer, TreeSet<TaskInfo>> entryAtPriority = iterator.next();
+ if (entryAtPriority.getKey() > forPriority) {
+ Iterator<TaskInfo> taskInfoIterator = entryAtPriority.getValue().iterator();
+ while (taskInfoIterator.hasNext() && preemptedCount < numTasksToPreempt) {
+ TaskInfo taskInfo = taskInfoIterator.next();
+ if (preemptHosts == null || preemptHosts.contains(taskInfo.assignedInstance.getHost())) {
+ // Candidate for preemption.
+ preemptedCount++;
+ LOG.info("preempting {} for task at priority {} with potentialHosts={}", taskInfo,
+ forPriority, potentialHosts == null ? "" : Arrays.toString(potentialHosts));
+ taskInfo.setPreemptedInfo(clock.getTime());
+ if (preemptedTaskList == null) {
+ preemptedTaskList = new LinkedList<>();
+ }
+ dagStats.registerTaskPreempted(taskInfo.assignedInstance.getHost());
+ preemptedTaskList.add(taskInfo);
+ registerPendingPreemption(taskInfo.assignedInstance.getHost());
+ // Remove from the runningTaskList
+ taskInfoIterator.remove();
+ }
+ }
+
+ // Remove entire priority level if it's been emptied.
+ if (entryAtPriority.getValue().isEmpty()) {
+ iterator.remove();
+ }
+ } else {
+ // No tasks qualify as preemptable
+ LOG.info("DBG: No tasks qualify as killable to schedule tasks at priority {}", forPriority);
+ break;
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ // Send out the preempted request outside of the lock.
+ if (preemptedTaskList != null) {
+ for (TaskInfo taskInfo : preemptedTaskList) {
+ LOG.info("DBG: Preempting task {}", taskInfo);
+ getContext().preemptContainer(taskInfo.containerId);
+ // Preemption will finally be registered as a deallocateTask as a result of preemptContainer
+ // That resets preemption info and allows additional tasks to be pre-empted if required.
+ }
+ }
+ // The schedule loop will be triggered again when the deallocateTask request comes in for the
+ // preempted task.
+ }
+
+ private void registerPendingPreemption(String host) {
+ writeLock.lock();
+ try {
+ pendingPreemptions.incrementAndGet();
+ MutableInt val = pendingPreemptionsPerHost.get(host);
+ if (val == null) {
+ val = new MutableInt(1);
+ pendingPreemptionsPerHost.put(host, val);
+ }
+ val.increment();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void unregisterPendingPreemption(String host) {
+ writeLock.lock();
+ try {
+ pendingPreemptions.decrementAndGet();
+ MutableInt val = pendingPreemptionsPerHost.get(host);
+ Preconditions.checkNotNull(val);
+ val.decrement();
+ // Not bothering with removing the entry. There's a limited number of hosts, and a good
+ // chance that the entry will make it back in when the AM is used for a long duration.
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private class NodeEnablerCallable implements Callable<Void> {
+
+ private AtomicBoolean isShutdown = new AtomicBoolean(false);
+ private static final long REFRESH_INTERVAL = 10000l;
+ long nextPollInterval = REFRESH_INTERVAL;
+ long lastRefreshTime;
+
+ @Override
+ public Void call() {
+
+ lastRefreshTime = System.currentTimeMillis();
+ while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
+ try {
+ while (true) {
+ NodeInfo nodeInfo = disabledNodesQueue.poll(nextPollInterval, TimeUnit.MILLISECONDS);
+ if (nodeInfo != null) {
+ long currentTime = System.currentTimeMillis();
+ // A node became available. Enable the node and try scheduling.
+ reenableDisabledNode(nodeInfo);
+ trySchedulingPendingTasks();
+
+ nextPollInterval -= (currentTime - lastRefreshTime);
+ }
+
+ if (nextPollInterval < 0 || nodeInfo == null) {
+ // timeout expired. Reset the poll interval and refresh nodes.
+ nextPollInterval = REFRESH_INTERVAL;
+ lastRefreshTime = System.currentTimeMillis();
+ // TODO Get rid of this polling once we have notificaitons from the registry sub-system
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Refreshing instances based on poll interval");
+ }
+ refreshInstances();
+ scanForNodeChanges();
+ }
+ }
+ } catch (InterruptedException e) {
+ if (isShutdown.get()) {
+ LOG.info("NodeEnabler thread interrupted after shutdown");
+ break;
+ } else {
+ LOG.warn("NodeEnabler thread interrupted without being shutdown");
+ throw new RuntimeException("NodeEnabler thread interrupted without being shutdown", e);
+ }
+ }
+ }
+ return null;
+ }
+
+ // Call this first, then send in an interrupt to the thread.
+ public void shutdown() {
+ isShutdown.set(true);
+ }
+ }
+
+ private void trySchedulingPendingTasks() {
+ scheduleLock.lock();
+ try {
+ pendingScheduleInvodations.set(true);
+ scheduleCondition.signal();
+ } finally {
+ scheduleLock.unlock();
+ }
+ }
+
+ private class SchedulerCallable implements Callable<Void> {
+ private AtomicBoolean isShutdown = new AtomicBoolean(false);
+
+ @Override
+ public Void call() {
+ while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
+ scheduleLock.lock();
+ try {
+ while (!pendingScheduleInvodations.get()) {
+ scheduleCondition.await();
+ }
+ } catch (InterruptedException e) {
+ if (isShutdown.get()) {
+ LOG.info("Scheduler thread interrupted after shutdown");
+ break;
+ } else {
+ LOG.warn("Scheduler thread interrupted without being shutdown");
+ throw new RuntimeException("Scheduler thread interrupted without being shutdown", e);
+ }
+ } finally {
+ scheduleLock.unlock();
+ }
+
+ // Set pending to false since scheduling is about to run. Any triggers up to this point
+ // will be handled in the next run.
+ // A new request may come in right after this is set to false, but before the actual scheduling.
+ // This will be handled in this run, but will cause an immediate run after, which is harmless.
+ pendingScheduleInvodations.set(false);
+ // Schedule outside of the scheduleLock - which should only be used to wait on the condition.
+ schedulePendingTasks();
+ }
+ return null;
+ }
+
+ // Call this first, then send in an interrupt to the thread.
+ public void shutdown() {
+ isShutdown.set(true);
+ }
+ }
+
+ @VisibleForTesting
+ static class NodeInfo implements Delayed {
+ private final NodeBlacklistConf blacklistConf;
+ final ServiceInstance serviceInstance;
+ private final Clock clock;
+
+ long expireTimeMillis = -1;
+ private long numSuccessfulTasks = 0;
+ private long numSuccessfulTasksAtLastBlacklist = -1;
+ float cumulativeBackoffFactor = 1.0f;
+
+ // Indicates whether a node had a recent communication failure.
+ private boolean hadCommFailure = false;
+
+ // Indicates whether a node is disabled - for whatever reason - commFailure, busy, etc.
+ private boolean disabled = false;
+
+ private int numPreemptedTasks = 0;
+ private int numScheduledTasks = 0;
+ private final int numSchedulableTasks;
+
+
+ /**
+ * Create a NodeInfo bound to a service instance
+ *
+ * @param serviceInstance the associated serviceInstance
+ * @param blacklistConf blacklist configuration
+ * @param clock clock to use to obtain timing information
+ * @param numSchedulableTasksConf number of schedulable tasks on the node. 0 represents auto
+ * detect based on the serviceInstance, -1 indicates indicates
+ * unlimited capacity
+ */
+ NodeInfo(ServiceInstance serviceInstance, NodeBlacklistConf blacklistConf, Clock clock, int numSchedulableTasksConf) {
+ Preconditions.checkArgument(numSchedulableTasksConf >= -1, "NumSchedulableTasks must be >=-1");
+ this.serviceInstance = serviceInstance;
+ this.blacklistConf = blacklistConf;
+ this.clock = clock;
+
+ if (numSchedulableTasksConf == 0) {
+ int pendingQueueuCapacity = 0;
+ String pendingQueueCapacityString = serviceInstance.getProperties()
+ .get(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting up node: " + serviceInstance + ", with available capacity=" +
+ serviceInstance.getResource().getVirtualCores() + ", pendingQueueCapacity=" +
+ pendingQueueCapacityString);
+ }
+
+ if (pendingQueueCapacityString != null) {
+ pendingQueueuCapacity = Integer.parseInt(pendingQueueCapacityString);
+ }
+ this.numSchedulableTasks = serviceInstance.getResource().getVirtualCores() + pendingQueueuCapacity;
+ } else {
+ this.numSchedulableTasks = numSchedulableTasksConf;
+ }
+ LOG.info("Setting up node: " + serviceInstance + " with schedulableCapacity=" + this.numSchedulableTasks);
+ }
+
+ ServiceInstance getServiceInstance() {
+ return serviceInstance;
+ }
+
+ void enableNode() {
+ expireTimeMillis = -1;
+ disabled = false;
+ hadCommFailure = false;
+ }
+
+ void disableNode(boolean commFailure) {
+ long duration = blacklistConf.minDelay;
+ long currentTime = clock.getTime();
+ this.hadCommFailure = commFailure;
+ disabled = true;
+ if (numSuccessfulTasksAtLastBlacklist == numSuccessfulTasks) {
+ // Relying on a task succeeding to reset the exponent.
+ // There's no notifications on whether a task gets accepted or not. That would be ideal to
+ // reset this.
+ cumulativeBackoffFactor = cumulativeBackoffFactor * blacklistConf.backoffFactor;
+ } else {
+ // Was able to execute something before the last blacklist. Reset the exponent.
+ cumulativeBackoffFactor = 1.0f;
+ }
+
+ long delayTime = (long) (duration * cumulativeBackoffFactor);
+ if (delayTime > blacklistConf.maxDelay) {
+ delayTime = blacklistConf.maxDelay;
+ }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Disabling instance " + serviceInstance + " for " + delayTime + " milli-seconds");
+ }
+ expireTimeMillis = currentTime + delayTime;
+ numSuccessfulTasksAtLastBlacklist = numSuccessfulTasks;
+ }
+
+ void registerTaskScheduled() {
+ numScheduledTasks++;
+ }
+
+ void registerTaskSuccess() {
+ numSuccessfulTasks++;
+ numScheduledTasks--;
+ }
+
+ void registerUnsuccessfulTaskEnd(boolean wasPreempted) {
+ numScheduledTasks--;
+ if (wasPreempted) {
+ numPreemptedTasks++;
+ }
+ }
+
+ public boolean isDisabled() {
+ return disabled;
+ }
+
+ public boolean hadCommFailure() {
+ return hadCommFailure;
+ }
+ /* Returning true does not guarantee that the task will run, considering other queries
+ may be running in the system. Also depends upon the capacity usage configuration
+ */
+ public boolean canAcceptTask() {
+ boolean result = !hadCommFailure && !disabled && serviceInstance.isAlive()
+ &&(numSchedulableTasks == -1 || ((numSchedulableTasks - numScheduledTasks) > 0));
+ LOG.info("canAcceptTask={}, numScheduledTasks={}, numSchedulableTasks={}, hadCommFailure={}, disabled={}, serviceInstance.isAlive={}", result, numScheduledTasks, numSchedulableTasks, hadCommFailure, disabled, serviceInstance.isAlive());
+ return result;
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit) {
+ return unit.convert(expireTimeMillis - clock.getTime(), TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed o) {
+ NodeInfo other = (NodeInfo) o;
+ if (other.expireTimeMillis > this.expireTimeMillis) {
+ return -1;
+ } else if (other.expireTimeMillis < this.expireTimeMillis) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "NodeInfo{" + "instance=" + serviceInstance
+ + ", expireTimeMillis=" + expireTimeMillis + ", numSuccessfulTasks=" + numSuccessfulTasks
+ + ", numSuccessfulTasksAtLastBlacklist=" + numSuccessfulTasksAtLastBlacklist
+ + ", cumulativeBackoffFactor=" + cumulativeBackoffFactor
+ + ", numSchedulableTasks=" + numSchedulableTasks
+ + ", numScheduledTasks=" + numScheduledTasks
+ + ", disabled=" + disabled
+ + ", commFailures=" + hadCommFailure
+ +'}';
+ }
+ }
+
+ @VisibleForTesting
+ static class StatsPerDag {
+ int numRequestedAllocations = 0;
+ int numRequestsWithLocation = 0;
+ int numRequestsWithoutLocation = 0;
+ int numTotalAllocations = 0;
+ int numLocalAllocations = 0;
+ int numNonLocalAllocations = 0;
+ int numAllocationsNoLocalityRequest = 0;
+ int numRejectedTasks = 0;
+ int numCommFailures = 0;
+ int numDelayedAllocations = 0;
+ int numPreemptedTasks = 0;
+ Map<String, AtomicInteger> localityBasedNumAllocationsPerHost = new HashMap<>();
+ Map<String, AtomicInteger> numAllocationsPerHost = new HashMap<>();
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("NumPreemptedTasks=").append(numPreemptedTasks).append(", ");
+ sb.append("NumRequestedAllocations=").append(numRequestedAllocations).append(", ");
+ sb.append("NumRequestsWithlocation=").append(numRequestsWithLocation).append(", ");
+ sb.append("NumLocalAllocations=").append(numLocalAllocations).append(",");
+ sb.append("NumNonLocalAllocations=").append(numNonLocalAllocations).append(",");
+ sb.append("NumTotalAllocations=").append(numTotalAllocations).append(",");
+ sb.append("NumRequestsWithoutLocation=").append(numRequestsWithoutLocation).append(", ");
+ sb.append("NumRejectedTasks=").append(numRejectedTasks).append(", ");
+ sb.append("NumCommFailures=").append(numCommFailures).append(", ");
+ sb.append("NumDelayedAllocations=").append(numDelayedAllocations).append(", ");
+ sb.append("LocalityBasedAllocationsPerHost=").append(localityBasedNumAllocationsPerHost)
+ .append(", ");
+ sb.append("NumAllocationsPerHost=").append(numAllocationsPerHost);
+ return sb.toString();
+ }
+
+ void registerTaskRequest(String[] requestedHosts, String[] requestedRacks) {
+ numRequestedAllocations++;
+ // TODO Change after HIVE-9987. For now, there's no rack matching.
+ if (requestedHosts != null && requestedHosts.length != 0) {
+ numRequestsWithLocation++;
+ } else {
+ numRequestsWithoutLocation++;
+ }
+ }
+
+ void registerTaskAllocated(String[] requestedHosts, String[] requestedRacks,
+ String allocatedHost) {
+ // TODO Change after HIVE-9987. For now, there's no rack matching.
+ if (requestedHosts != null && requestedHosts.length != 0) {
+ Set<String> requestedHostSet = new HashSet<>(Arrays.asList(requestedHosts));
+ if (requestedHostSet.contains(allocatedHost)) {
+ numLocalAllocations++;
+ _registerAllocationInHostMap(allocatedHost, localityBasedNumAllocationsPerHost);
+ } else {
+ numNonLocalAllocations++;
+ }
+ } else {
+ numAllocationsNoLocalityRequest++;
+ }
+ numTotalAllocations++;
+ _registerAllocationInHostMap(allocatedHost, numAllocationsPerHost);
+ }
+
+ void registerTaskPreempted(String host) {
+ numPreemptedTasks++;
+ }
+
+ void registerCommFailure(String host) {
+ numCommFailures++;
+ }
+
+ void registerTaskRejected(String host) {
+ numRejectedTasks++;
+ }
+
+ void registerDelayedAllocation() {
+ numDelayedAllocations++;
+ }
+
+ private void _registerAllocationInHostMap(String host, Map<String, AtomicInteger> hostMap) {
+ AtomicInteger val = hostMap.get(host);
+ if (val == null) {
+ val = new AtomicInteger(0);
+ hostMap.put(host, val);
+ }
+ val.incrementAndGet();
+ }
+ }
+
+ private static class TaskInfo {
+ // IDs used to ensure two TaskInfos are different without using the underlying task instance.
+ // Required for insertion into a TreeMap
+ static final AtomicLong ID_GEN = new AtomicLong(0);
+ final long uniqueId;
+ final Object task;
+ final Object clientCookie;
+ final Priority priority;
+ final Resource capability;
+ final String[] requestedHosts;
+ final String[] requestedRacks;
+ final long requestTime;
+ long startTime;
+ long preemptTime;
+ ContainerId containerId;
+ ServiceInstance assignedInstance;
+ private boolean assigned = false;
+ private boolean preempted = false;
+
+ private int numAssignAttempts = 0;
+
+ // TaskInfo instances for two different tasks will not be the same. Only a single instance should
+ // ever be created for a taskAttempt
+ public TaskInfo(Object task, Object clientCookie, Priority priority, Resource capability,
+ String[] hosts, String[] racks, long requestTime) {
+ this.task = task;
+ this.clientCookie = clientCookie;
+ this.priority = priority;
+ this.capability = capability;
+ this.requestedHosts = hosts;
+ this.requestedRacks = racks;
+ this.requestTime = requestTime;
+ this.uniqueId = ID_GEN.getAndIncrement();
+ }
+
+ void setAssignmentInfo(ServiceInstance instance, ContainerId containerId, long startTime) {
+ this.assignedInstance = instance;
+ this.containerId = containerId;
+ this.startTime = startTime;
+ assigned = true;
+ }
+
+ void setPreemptedInfo(long preemptTime) {
+ this.preempted = true;
+ this.assigned = false;
+ this.preemptTime = preemptTime;
+ }
+
+ void triedAssigningTask() {
+ numAssignAttempts++;
+ }
+
+ int getNumPreviousAssignAttempts() {
+ return numAssignAttempts;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ TaskInfo taskInfo = (TaskInfo) o;
+
+ if (uniqueId != taskInfo.uniqueId) {
+ return false;
+ }
+ return task.equals(taskInfo.task);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (uniqueId ^ (uniqueId >>> 32));
+ result = 31 * result + task.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "TaskInfo{" +
+ "task=" + task +
+ ", priority=" + priority +
+ ", startTime=" + startTime +
+ ", containerId=" + containerId +
+ ", assignedInstance=" + assignedInstance +
+ ", uniqueId=" + uniqueId +
+ '}';
+ }
+ }
+
+ // Newer tasks first.
+ private static class TaskStartComparator implements Comparator<TaskInfo> {
+
+ @Override
+ public int compare(TaskInfo o1, TaskInfo o2) {
+ if (o1.startTime > o2.startTime) {
+ return -1;
+ } else if (o1.startTime < o2.startTime) {
+ return 1;
+ } else {
+ // Comparing on time is not sufficient since two may be created at the same time,
+ // in which case inserting into a TreeSet/Map would break
+ if (o1.uniqueId > o2.uniqueId) {
+ return -1;
+ } else if (o1.uniqueId < o2.uniqueId) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+ }
+ }
+
+ private static class SelectHostResult {
+ final NodeServiceInstancePair nodeServiceInstancePair;
+ final ScheduleResult scheduleResult;
+
+ SelectHostResult(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
+ this.nodeServiceInstancePair = new NodeServiceInstancePair(serviceInstance, nodeInfo);
+ this.scheduleResult = ScheduleResult.SCHEDULED;
+ }
+
+ SelectHostResult(ScheduleResult scheduleResult) {
+ this.nodeServiceInstancePair = null;
+ this.scheduleResult = scheduleResult;
+ }
+ }
+
+ private static final SelectHostResult SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY =
+ new SelectHostResult(ScheduleResult.INADEQUATE_TOTAL_RESOURCES);
+ private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_LOCALITY =
+ new SelectHostResult(ScheduleResult.DELAYED_LOCALITY);
+ private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_RESOURCES =
+ new SelectHostResult(ScheduleResult.DELAYED_RESOURCES);
+
+ private static class NodeServiceInstancePair {
+ final NodeInfo nodeInfo;
+ final ServiceInstance serviceInstance;
+
+ private NodeServiceInstancePair(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
+ this.serviceInstance = serviceInstance;
+ this.nodeInfo = nodeInfo;
+ }
+
+ public ServiceInstance getServiceInstance() {
+ return serviceInstance;
+ }
+
+ public NodeInfo getNodeInfo() {
+ return nodeInfo;
+ }
+ }
+
+ private static final class NodeBlacklistConf {
+ private final long minDelay;
+ private final long maxDelay;
+ private final float backoffFactor;
+
+ public NodeBlacklistConf(long minDelay, long maxDelay, float backoffFactor) {
+ this.minDelay = minDelay;
+ this.maxDelay = maxDelay;
+ this.backoffFactor = backoffFactor;
+ }
+
+ @Override
+ public String toString() {
+ return "NodeBlacklistConf{" +
+ "minDelay=" + minDelay +
+ ", maxDelay=" + maxDelay +
+ ", backoffFactor=" + backoffFactor +
+ '}';
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java
new file mode 100644
index 0000000..2c3e53c
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTezUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.tez.mapreduce.input.MultiMRInput;
+
+@InterfaceAudience.Private
+public class LlapTezUtils {
+ public static boolean isSourceOfInterest(String inputClassName) {
+ // MRInput is not of interest since it'll always be ready.
+ return !(inputClassName.equals(MRInputLegacy.class.getName()) || inputClassName.equals(
+ MultiMRInput.class.getName()) || inputClassName.equals(MRInput.class.getName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
new file mode 100644
index 0000000..4102f5b
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
+import org.apache.hadoop.security.authorize.Service;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
+
+public class LlapUmbilicalPolicyProvider extends TezAMPolicyProvider {
+
+ private static Service[] services;
+ private static final Object servicesLock = new Object();
+
+ @Override
+ public Service[] getServices() {
+ if (services != null) return services;
+ synchronized (servicesLock) {
+ if (services != null) return services;
+ Service[] parentSvc = super.getServices();
+ Service[] result = Arrays.copyOf(parentSvc, parentSvc.length + 1);
+ result[parentSvc.length] = new Service(
+ TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL,
+ LlapTaskUmbilicalProtocol.class);
+ return (services = result);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
new file mode 100644
index 0000000..d8f7574
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins.helpers;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
+import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.tez.Converters;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.runtime.api.impl.InputSpec;
+
+public class SourceStateTracker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SourceStateTracker.class);
+
+ private final TaskCommunicatorContext taskCommunicatorContext;
+ private final LlapTaskCommunicator taskCommunicator;
+
+ private final QueryIdentifierProto BASE_QUERY_IDENTIFIER;
+
+ // Tracks vertices for which notifications have been registered
+ private final Set<String> notificationRegisteredVertices = new HashSet<>();
+
+ private final Map<String, SourceInfo> sourceInfoMap = new HashMap<>();
+ private final Map<LlapNodeId, NodeInfo> nodeInfoMap = new HashMap<>();
+
+ private volatile QueryIdentifierProto currentQueryIdentifier;
+
+ public SourceStateTracker(TaskCommunicatorContext taskCommunicatorContext,
+ LlapTaskCommunicator taskCommunicator) {
+ this.taskCommunicatorContext = taskCommunicatorContext;
+ this.taskCommunicator = taskCommunicator;
+ BASE_QUERY_IDENTIFIER = QueryIdentifierProto.newBuilder()
+ .setAppIdentifier(taskCommunicatorContext.getCurrentAppIdentifier()).build();
+ }
+
+ /**
+ * To be invoked after each DAG completes.
+ */
+ public synchronized void resetState(int newDagId) {
+ sourceInfoMap.clear();
+ nodeInfoMap.clear();
+ notificationRegisteredVertices.clear();
+ this.currentQueryIdentifier =
+ QueryIdentifierProto.newBuilder(BASE_QUERY_IDENTIFIER).setDagIdentifier(newDagId).build();
+ }
+
+ /**
+ * Used to register a task for state updates. Effectively registers for state updates to go to the specific node.
+ * @param host
+ * @param port
+ * @param inputSpecList
+ */
+ public synchronized void registerTaskForStateUpdates(String host, int port,
+ List<InputSpec> inputSpecList) {
+
+ // Add tracking information. Check if source state already known and send out an update if it is.
+
+ List<String> sourcesOfInterest = getSourceInterestList(inputSpecList);
+ if (sourcesOfInterest != null && !sourcesOfInterest.isEmpty()) {
+ LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+ NodeInfo nodeInfo = getNodeInfo(nodeId);
+
+ // Set up the data structures, before any notifications come in.
+ for (String src : sourcesOfInterest) {
+ VertexState oldStateForNode = nodeInfo.getLastKnownStateForSource(src);
+ if (oldStateForNode == null) {
+ // Not registered for this node.
+ // Register and send state if it is successful.
+ SourceInfo srcInfo = getSourceInfo(src);
+ srcInfo.addNode(nodeId);
+
+ nodeInfo.addSource(src, srcInfo.lastKnownState);
+ if (srcInfo.lastKnownState == VertexState.SUCCEEDED) {
+ sendStateUpdateToNode(nodeId, src, srcInfo.lastKnownState);
+ }
+
+ } else {
+ // Already registered to send updates to this node for the specific source.
+ // Nothing to do for now, unless tracking tasks at a later point.
+ }
+
+ // Setup for actual notifications, if not already done for a previous task.
+ maybeRegisterForVertexUpdates(src);
+ }
+ } else {
+ // Don't need to track anything for this task. No new notifications, etc.
+ }
+ }
+
+ /**
+ * Handled notifications on state updates for sources
+ * @param sourceName
+ * @param sourceState
+ */
+ public synchronized void sourceStateUpdated(String sourceName, VertexState sourceState) {
+ SourceInfo sourceInfo = getSourceInfo(sourceName);
+ // Update source info if the state is SUCCEEDED
+ if (sourceState == VertexState.SUCCEEDED) {
+ sourceInfo.numCompletedTasks = taskCommunicatorContext.getVertexCompletedTaskCount(sourceName);
+ sourceInfo.numTasks = taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
+ }
+ sourceInfo.lastKnownState = sourceState;
+ // Checking state per node for future failure handling scenarios, where an update
+ // to a single node may fail.
+ for (LlapNodeId nodeId : sourceInfo.getInterestedNodes()) {
+ NodeInfo nodeInfo = nodeInfoMap.get(nodeId);
+ VertexState lastStateForNode = nodeInfo.getLastKnownStateForSource(sourceName);
+ // Send only if the state has changed.
+ if (lastStateForNode != sourceState) {
+ nodeInfo.setLastKnownStateForSource(sourceName, sourceState);
+ sendStateUpdateToNode(nodeId, sourceName, sourceState);
+ }
+ }
+ }
+
+
+ // Assumes serialized DAGs within an AM, and a reset of structures after each DAG completes.
+ /**
+ * Constructs FragmentRuntimeInfo for scheduling within LLAP daemons.
+ * Also caches state based on state updates.
+ * @param vertexName
+ * @param fragmentNumber
+ * @param priority
+ * @return
+ */
+ public synchronized FragmentRuntimeInfo getFragmentRuntimeInfo(String vertexName, int fragmentNumber,
+ int priority) {
+ FragmentRuntimeInfo.Builder builder = FragmentRuntimeInfo.newBuilder();
+ maybeRegisterForVertexUpdates(vertexName);
+
+ MutableInt totalTaskCount = new MutableInt(0);
+ MutableInt completedTaskCount = new MutableInt(0);
+ computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, vertexName);
+
+ builder.setNumSelfAndUpstreamCompletedTasks(completedTaskCount.intValue());
+ builder.setNumSelfAndUpstreamTasks(totalTaskCount.intValue());
+ builder.setDagStartTime(taskCommunicatorContext.getDagStartTime());
+ builder.setWithinDagPriority(priority);
+ builder.setFirstAttemptStartTime(taskCommunicatorContext.getFirstAttemptStartTime(vertexName, fragmentNumber));
+ builder.setCurrentAttemptStartTime(System.currentTimeMillis());
+ return builder.build();
+ }
+
+ private void computeUpstreamTaskCounts(MutableInt completedTaskCount, MutableInt totalTaskCount, String sourceName) {
+ SourceInfo sourceInfo = getSourceInfo(sourceName);
+ if (sourceInfo.lastKnownState == VertexState.SUCCEEDED) {
+ // Some of the information in the source is complete. Don't need to fetch it from the context.
+ completedTaskCount.add(sourceInfo.numCompletedTasks);
+ totalTaskCount.add(sourceInfo.numTasks);
+ } else {
+ completedTaskCount.add(taskCommunicatorContext.getVertexCompletedTaskCount(sourceName));
+ int totalCount =taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
+ // Uninitialized vertices will report count as 0.
+ totalCount = totalCount == -1 ? 0 : totalCount;
+ totalTaskCount.add(totalCount);
+ }
+
+ // Walk through all the source vertices
+ for (String up : taskCommunicatorContext.getInputVertexNames(sourceName)) {
+ computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, up);
+ }
+ }
+
+ private static class SourceInfo {
+
+ // Always start in the running state. Requests for state updates will be sent out after registration.
+ private VertexState lastKnownState = VertexState.RUNNING;
+
+ // Used for sending notifications about a vertex completed. For canFinish
+ // Can be converted to a Tez event, if this is sufficient to decide on pre-emption
+ private final List<LlapNodeId> interestedNodes = new LinkedList<>();
+
+ // Used for sending information for scheduling priority.
+ private int numTasks;
+ private int numCompletedTasks;
+
+ void addNode(LlapNodeId nodeId) {
+ interestedNodes.add(nodeId);
+ }
+
+ List<LlapNodeId> getInterestedNodes() {
+ return this.interestedNodes;
+ }
+
+
+
+ }
+
+ private synchronized SourceInfo getSourceInfo(String srcName) {
+ SourceInfo sourceInfo = sourceInfoMap.get(srcName);
+ if (sourceInfo == null) {
+ sourceInfo = new SourceInfo();
+ sourceInfoMap.put(srcName, sourceInfo);
+ }
+ return sourceInfo;
+ }
+
+
+ private static class NodeInfo {
+ private final Map<String, VertexState> sourcesOfInterest = new HashMap<>();
+
+ void addSource(String srcName, VertexState sourceState) {
+ sourcesOfInterest.put(srcName, sourceState);
+ }
+
+ VertexState getLastKnownStateForSource(String src) {
+ return sourcesOfInterest.get(src);
+ }
+
+ void setLastKnownStateForSource(String src, VertexState state) {
+ sourcesOfInterest.put(src, state);
+ }
+ }
+
+ private synchronized NodeInfo getNodeInfo(LlapNodeId llapNodeId) {
+ NodeInfo nodeInfo = nodeInfoMap.get(llapNodeId);
+ if (nodeInfo == null) {
+ nodeInfo = new NodeInfo();
+ nodeInfoMap.put(llapNodeId, nodeInfo);
+ }
+ return nodeInfo;
+ }
+
+
+ private List<String> getSourceInterestList(List<InputSpec> inputSpecList) {
+ List<String> sourcesOfInterest = Collections.emptyList();
+ if (inputSpecList != null) {
+ boolean alreadyFound = false;
+ for (InputSpec inputSpec : inputSpecList) {
+ if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) {
+ if (!alreadyFound) {
+ alreadyFound = true;
+ sourcesOfInterest = new LinkedList<>();
+ }
+ sourcesOfInterest.add(inputSpec.getSourceVertexName());
+ }
+ }
+ }
+ return sourcesOfInterest;
+ }
+
+
+ private void maybeRegisterForVertexUpdates(String sourceName) {
+ if (!notificationRegisteredVertices.contains(sourceName)) {
+ notificationRegisteredVertices.add(sourceName);
+ taskCommunicatorContext.registerForVertexStateUpdates(sourceName, EnumSet.of(
+ VertexState.RUNNING, VertexState.SUCCEEDED));
+ }
+ }
+
+
+
+ void sendStateUpdateToNode(LlapNodeId nodeId, String sourceName, VertexState state) {
+ taskCommunicator.sendStateUpdate(nodeId.getHostname(), nodeId.getPort(),
+ SourceStateUpdatedRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifier)
+ .setSrcName(sourceName).setState(Converters.fromVertexState(state)).build());
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
new file mode 100644
index 0000000..8f3d104
--- /dev/null
+++ b/llap-tez/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.junit.Test;
+
+public class TestLlapTaskCommunicator {
+
+ @Test (timeout = 5000)
+ public void testEntityTracker1() {
+ LlapTaskCommunicator.EntityTracker entityTracker = new LlapTaskCommunicator.EntityTracker();
+
+ String host1 = "host1";
+ String host2 = "host2";
+ String host3 = "host3";
+ int port = 1451;
+
+
+ // Simple container registration and un-registration without any task attempt being involved.
+ ContainerId containerId101 = constructContainerId(101);
+ entityTracker.registerContainer(containerId101, host1, port);
+ assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId101));
+
+ entityTracker.unregisterContainer(containerId101);
+ assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
+ assertNull(entityTracker.getNodeIdForContainer(containerId101));
+ assertEquals(0, entityTracker.nodeMap.size());
+ assertEquals(0, entityTracker.attemptToNodeMap.size());
+ assertEquals(0, entityTracker.containerToNodeMap.size());
+
+
+ // Simple task registration and un-registration.
+ ContainerId containerId1 = constructContainerId(1);
+ TezTaskAttemptID taskAttemptId1 = constructTaskAttemptId(1);
+ entityTracker.registerTaskAttempt(containerId1, taskAttemptId1, host1, port);
+ assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId1));
+ assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
+
+ entityTracker.unregisterTaskAttempt(taskAttemptId1);
+ assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
+ assertNull(entityTracker.getNodeIdForContainer(containerId1));
+ assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
+ assertEquals(0, entityTracker.nodeMap.size());
+ assertEquals(0, entityTracker.attemptToNodeMap.size());
+ assertEquals(0, entityTracker.containerToNodeMap.size());
+
+ // Register taskAttempt, unregister container. TaskAttempt should also be unregistered
+ ContainerId containerId201 = constructContainerId(201);
+ TezTaskAttemptID taskAttemptId201 = constructTaskAttemptId(201);
+ entityTracker.registerTaskAttempt(containerId201, taskAttemptId201, host1, port);
+ assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId201));
+ assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
+
+ entityTracker.unregisterContainer(containerId201);
+ assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
+ assertNull(entityTracker.getNodeIdForContainer(containerId201));
+ assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
+ assertEquals(0, entityTracker.nodeMap.size());
+ assertEquals(0, entityTracker.attemptToNodeMap.size());
+ assertEquals(0, entityTracker.containerToNodeMap.size());
+
+ entityTracker.unregisterTaskAttempt(taskAttemptId201); // No errors
+ }
+
+
+ private ContainerId constructContainerId(int id) {
+ ContainerId containerId = mock(ContainerId.class);
+ doReturn(id).when(containerId).getId();
+ doReturn((long)id).when(containerId).getContainerId();
+ return containerId;
+ }
+
+ private TezTaskAttemptID constructTaskAttemptId(int id) {
+ TezTaskAttemptID taskAttemptId = mock(TezTaskAttemptID.class);
+ doReturn(id).when(taskAttemptId).getId();
+ return taskAttemptId;
+ }
+
+}
[09/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
----------------------------------------------------------------------
diff --git a/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java b/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
new file mode 100644
index 0000000..4ab7b32
--- /dev/null
+++ b/llap-common/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
@@ -0,0 +1,14553 @@
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+// source: LlapDaemonProtocol.proto
+
+package org.apache.hadoop.hive.llap.daemon.rpc;
+
+public final class LlapDaemonProtocolProtos {
+ private LlapDaemonProtocolProtos() {}
+ public static void registerAllExtensions(
+ com.google.protobuf.ExtensionRegistry registry) {
+ }
+ /**
+ * Protobuf enum {@code SourceStateProto}
+ */
+ public enum SourceStateProto
+ implements com.google.protobuf.ProtocolMessageEnum {
+ /**
+ * <code>S_SUCCEEDED = 1;</code>
+ */
+ S_SUCCEEDED(0, 1),
+ /**
+ * <code>S_RUNNING = 2;</code>
+ */
+ S_RUNNING(1, 2),
+ ;
+
+ /**
+ * <code>S_SUCCEEDED = 1;</code>
+ */
+ public static final int S_SUCCEEDED_VALUE = 1;
+ /**
+ * <code>S_RUNNING = 2;</code>
+ */
+ public static final int S_RUNNING_VALUE = 2;
+
+
+ public final int getNumber() { return value; }
+
+ public static SourceStateProto valueOf(int value) {
+ switch (value) {
+ case 1: return S_SUCCEEDED;
+ case 2: return S_RUNNING;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>() {
+ public SourceStateProto findValueByNumber(int number) {
+ return SourceStateProto.valueOf(number);
+ }
+ };
+
+ public final com.google.protobuf.Descriptors.EnumValueDescriptor
+ getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+ }
+ public final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptorForType() {
+ return getDescriptor();
+ }
+ public static final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(0);
+ }
+
+ private static final SourceStateProto[] VALUES = values();
+
+ public static SourceStateProto valueOf(
+ com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+ throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+ }
+
+ private final int index;
+ private final int value;
+
+ private SourceStateProto(int index, int value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:SourceStateProto)
+ }
+
+ /**
+ * Protobuf enum {@code SubmissionStateProto}
+ */
+ public enum SubmissionStateProto
+ implements com.google.protobuf.ProtocolMessageEnum {
+ /**
+ * <code>ACCEPTED = 1;</code>
+ */
+ ACCEPTED(0, 1),
+ /**
+ * <code>REJECTED = 2;</code>
+ */
+ REJECTED(1, 2),
+ /**
+ * <code>EVICTED_OTHER = 3;</code>
+ */
+ EVICTED_OTHER(2, 3),
+ ;
+
+ /**
+ * <code>ACCEPTED = 1;</code>
+ */
+ public static final int ACCEPTED_VALUE = 1;
+ /**
+ * <code>REJECTED = 2;</code>
+ */
+ public static final int REJECTED_VALUE = 2;
+ /**
+ * <code>EVICTED_OTHER = 3;</code>
+ */
+ public static final int EVICTED_OTHER_VALUE = 3;
+
+
+ public final int getNumber() { return value; }
+
+ public static SubmissionStateProto valueOf(int value) {
+ switch (value) {
+ case 1: return ACCEPTED;
+ case 2: return REJECTED;
+ case 3: return EVICTED_OTHER;
+ default: return null;
+ }
+ }
+
+ public static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
+ internalGetValueMap() {
+ return internalValueMap;
+ }
+ private static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
+ internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>() {
+ public SubmissionStateProto findValueByNumber(int number) {
+ return SubmissionStateProto.valueOf(number);
+ }
+ };
+
+ public final com.google.protobuf.Descriptors.EnumValueDescriptor
+ getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+ }
+ public final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptorForType() {
+ return getDescriptor();
+ }
+ public static final com.google.protobuf.Descriptors.EnumDescriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(1);
+ }
+
+ private static final SubmissionStateProto[] VALUES = values();
+
+ public static SubmissionStateProto valueOf(
+ com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+ throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+ }
+
+ private final int index;
+ private final int value;
+
+ private SubmissionStateProto(int index, int value) {
+ this.index = index;
+ this.value = value;
+ }
+
+ // @@protoc_insertion_point(enum_scope:SubmissionStateProto)
+ }
+
+ public interface UserPayloadProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional bytes user_payload = 1;
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ boolean hasUserPayload();
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ com.google.protobuf.ByteString getUserPayload();
+
+ // optional int32 version = 2;
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ boolean hasVersion();
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ int getVersion();
+ }
+ /**
+ * Protobuf type {@code UserPayloadProto}
+ */
+ public static final class UserPayloadProto extends
+ com.google.protobuf.GeneratedMessage
+ implements UserPayloadProtoOrBuilder {
+ // Use UserPayloadProto.newBuilder() to construct.
+ private UserPayloadProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private UserPayloadProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final UserPayloadProto defaultInstance;
+ public static UserPayloadProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public UserPayloadProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private UserPayloadProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ userPayload_ = input.readBytes();
+ break;
+ }
+ case 16: {
+ bitField0_ |= 0x00000002;
+ version_ = input.readInt32();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<UserPayloadProto> PARSER =
+ new com.google.protobuf.AbstractParser<UserPayloadProto>() {
+ public UserPayloadProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new UserPayloadProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<UserPayloadProto> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional bytes user_payload = 1;
+ public static final int USER_PAYLOAD_FIELD_NUMBER = 1;
+ private com.google.protobuf.ByteString userPayload_;
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ public boolean hasUserPayload() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ public com.google.protobuf.ByteString getUserPayload() {
+ return userPayload_;
+ }
+
+ // optional int32 version = 2;
+ public static final int VERSION_FIELD_NUMBER = 2;
+ private int version_;
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ public boolean hasVersion() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ public int getVersion() {
+ return version_;
+ }
+
+ private void initFields() {
+ userPayload_ = com.google.protobuf.ByteString.EMPTY;
+ version_ = 0;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, userPayload_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeInt32(2, version_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, userPayload_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(2, version_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) obj;
+
+ boolean result = true;
+ result = result && (hasUserPayload() == other.hasUserPayload());
+ if (hasUserPayload()) {
+ result = result && getUserPayload()
+ .equals(other.getUserPayload());
+ }
+ result = result && (hasVersion() == other.hasVersion());
+ if (hasVersion()) {
+ result = result && (getVersion()
+ == other.getVersion());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasUserPayload()) {
+ hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
+ hash = (53 * hash) + getUserPayload().hashCode();
+ }
+ if (hasVersion()) {
+ hash = (37 * hash) + VERSION_FIELD_NUMBER;
+ hash = (53 * hash) + getVersion();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code UserPayloadProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ userPayload_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ version_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getDefaultInstanceForType() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto build() {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto buildPartial() {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.userPayload_ = userPayload_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.version_ = version_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) {
+ return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other) {
+ if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) return this;
+ if (other.hasUserPayload()) {
+ setUserPayload(other.getUserPayload());
+ }
+ if (other.hasVersion()) {
+ setVersion(other.getVersion());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional bytes user_payload = 1;
+ private com.google.protobuf.ByteString userPayload_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ public boolean hasUserPayload() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ public com.google.protobuf.ByteString getUserPayload() {
+ return userPayload_;
+ }
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ public Builder setUserPayload(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ userPayload_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes user_payload = 1;</code>
+ */
+ public Builder clearUserPayload() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ userPayload_ = getDefaultInstance().getUserPayload();
+ onChanged();
+ return this;
+ }
+
+ // optional int32 version = 2;
+ private int version_ ;
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ public boolean hasVersion() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ public int getVersion() {
+ return version_;
+ }
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ public Builder setVersion(int value) {
+ bitField0_ |= 0x00000002;
+ version_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int32 version = 2;</code>
+ */
+ public Builder clearVersion() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ version_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:UserPayloadProto)
+ }
+
+ static {
+ defaultInstance = new UserPayloadProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:UserPayloadProto)
+ }
+
+ public interface EntityDescriptorProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional string class_name = 1;
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ boolean hasClassName();
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ java.lang.String getClassName();
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getClassNameBytes();
+
+ // optional .UserPayloadProto user_payload = 2;
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ boolean hasUserPayload();
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload();
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder();
+
+ // optional bytes history_text = 3;
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ boolean hasHistoryText();
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ com.google.protobuf.ByteString getHistoryText();
+ }
+ /**
+ * Protobuf type {@code EntityDescriptorProto}
+ */
+ public static final class EntityDescriptorProto extends
+ com.google.protobuf.GeneratedMessage
+ implements EntityDescriptorProtoOrBuilder {
+ // Use EntityDescriptorProto.newBuilder() to construct.
+ private EntityDescriptorProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private EntityDescriptorProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final EntityDescriptorProto defaultInstance;
+ public static EntityDescriptorProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public EntityDescriptorProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private EntityDescriptorProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ className_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ subBuilder = userPayload_.toBuilder();
+ }
+ userPayload_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(userPayload_);
+ userPayload_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000002;
+ break;
+ }
+ case 26: {
+ bitField0_ |= 0x00000004;
+ historyText_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<EntityDescriptorProto> PARSER =
+ new com.google.protobuf.AbstractParser<EntityDescriptorProto>() {
+ public EntityDescriptorProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new EntityDescriptorProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<EntityDescriptorProto> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional string class_name = 1;
+ public static final int CLASS_NAME_FIELD_NUMBER = 1;
+ private java.lang.Object className_;
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public boolean hasClassName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public java.lang.String getClassName() {
+ java.lang.Object ref = className_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ className_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getClassNameBytes() {
+ java.lang.Object ref = className_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ className_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional .UserPayloadProto user_payload = 2;
+ public static final int USER_PAYLOAD_FIELD_NUMBER = 2;
+ private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_;
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public boolean hasUserPayload() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
+ return userPayload_;
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
+ return userPayload_;
+ }
+
+ // optional bytes history_text = 3;
+ public static final int HISTORY_TEXT_FIELD_NUMBER = 3;
+ private com.google.protobuf.ByteString historyText_;
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ public boolean hasHistoryText() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ public com.google.protobuf.ByteString getHistoryText() {
+ return historyText_;
+ }
+
+ private void initFields() {
+ className_ = "";
+ userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+ historyText_ = com.google.protobuf.ByteString.EMPTY;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getClassNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeMessage(2, userPayload_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeBytes(3, historyText_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getClassNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(2, userPayload_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(3, historyText_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) obj;
+
+ boolean result = true;
+ result = result && (hasClassName() == other.hasClassName());
+ if (hasClassName()) {
+ result = result && getClassName()
+ .equals(other.getClassName());
+ }
+ result = result && (hasUserPayload() == other.hasUserPayload());
+ if (hasUserPayload()) {
+ result = result && getUserPayload()
+ .equals(other.getUserPayload());
+ }
+ result = result && (hasHistoryText() == other.hasHistoryText());
+ if (hasHistoryText()) {
+ result = result && getHistoryText()
+ .equals(other.getHistoryText());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasClassName()) {
+ hash = (37 * hash) + CLASS_NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getClassName().hashCode();
+ }
+ if (hasUserPayload()) {
+ hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
+ hash = (53 * hash) + getUserPayload().hashCode();
+ }
+ if (hasHistoryText()) {
+ hash = (37 * hash) + HISTORY_TEXT_FIELD_NUMBER;
+ hash = (53 * hash) + getHistoryText().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code EntityDescriptorProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getUserPayloadFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ className_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ if (userPayloadBuilder_ == null) {
+ userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+ } else {
+ userPayloadBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ historyText_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getDefaultInstanceForType() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto build() {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto buildPartial() {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.className_ = className_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ if (userPayloadBuilder_ == null) {
+ result.userPayload_ = userPayload_;
+ } else {
+ result.userPayload_ = userPayloadBuilder_.build();
+ }
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.historyText_ = historyText_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) {
+ return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other) {
+ if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) return this;
+ if (other.hasClassName()) {
+ bitField0_ |= 0x00000001;
+ className_ = other.className_;
+ onChanged();
+ }
+ if (other.hasUserPayload()) {
+ mergeUserPayload(other.getUserPayload());
+ }
+ if (other.hasHistoryText()) {
+ setHistoryText(other.getHistoryText());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional string class_name = 1;
+ private java.lang.Object className_ = "";
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public boolean hasClassName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public java.lang.String getClassName() {
+ java.lang.Object ref = className_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ className_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getClassNameBytes() {
+ java.lang.Object ref = className_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ className_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public Builder setClassName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ className_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public Builder clearClassName() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ className_ = getDefaultInstance().getClassName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string class_name = 1;</code>
+ */
+ public Builder setClassNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ className_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional .UserPayloadProto user_payload = 2;
+ private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder> userPayloadBuilder_;
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public boolean hasUserPayload() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
+ if (userPayloadBuilder_ == null) {
+ return userPayload_;
+ } else {
+ return userPayloadBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public Builder setUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
+ if (userPayloadBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ userPayload_ = value;
+ onChanged();
+ } else {
+ userPayloadBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public Builder setUserPayload(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder builderForValue) {
+ if (userPayloadBuilder_ == null) {
+ userPayload_ = builderForValue.build();
+ onChanged();
+ } else {
+ userPayloadBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public Builder mergeUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
+ if (userPayloadBuilder_ == null) {
+ if (((bitField0_ & 0x00000002) == 0x00000002) &&
+ userPayload_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) {
+ userPayload_ =
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder(userPayload_).mergeFrom(value).buildPartial();
+ } else {
+ userPayload_ = value;
+ }
+ onChanged();
+ } else {
+ userPayloadBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public Builder clearUserPayload() {
+ if (userPayloadBuilder_ == null) {
+ userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
+ onChanged();
+ } else {
+ userPayloadBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder getUserPayloadBuilder() {
+ bitField0_ |= 0x00000002;
+ onChanged();
+ return getUserPayloadFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
+ if (userPayloadBuilder_ != null) {
+ return userPayloadBuilder_.getMessageOrBuilder();
+ } else {
+ return userPayload_;
+ }
+ }
+ /**
+ * <code>optional .UserPayloadProto user_payload = 2;</code>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder>
+ getUserPayloadFieldBuilder() {
+ if (userPayloadBuilder_ == null) {
+ userPayloadBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder>(
+ userPayload_,
+ getParentForChildren(),
+ isClean());
+ userPayload_ = null;
+ }
+ return userPayloadBuilder_;
+ }
+
+ // optional bytes history_text = 3;
+ private com.google.protobuf.ByteString historyText_ = com.google.protobuf.ByteString.EMPTY;
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ public boolean hasHistoryText() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ public com.google.protobuf.ByteString getHistoryText() {
+ return historyText_;
+ }
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ public Builder setHistoryText(com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ historyText_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bytes history_text = 3;</code>
+ */
+ public Builder clearHistoryText() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ historyText_ = getDefaultInstance().getHistoryText();
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:EntityDescriptorProto)
+ }
+
+ static {
+ defaultInstance = new EntityDescriptorProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:EntityDescriptorProto)
+ }
+
+ public interface IOSpecProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // optional string connected_vertex_name = 1;
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ boolean hasConnectedVertexName();
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ java.lang.String getConnectedVertexName();
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getConnectedVertexNameBytes();
+
+ // optional .EntityDescriptorProto io_descriptor = 2;
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ boolean hasIoDescriptor();
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor();
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder();
+
+ // optional int32 physical_edge_count = 3;
+ /**
+ * <code>optional int32 physical_edge_count = 3;</code>
+ */
+ boolean hasPhysicalEdgeCount();
+ /**
+ * <code>optional int32 physical_edge_count = 3;</code>
+ */
+ int getPhysicalEdgeCount();
+ }
+ /**
+ * Protobuf type {@code IOSpecProto}
+ */
+ public static final class IOSpecProto extends
+ com.google.protobuf.GeneratedMessage
+ implements IOSpecProtoOrBuilder {
+ // Use IOSpecProto.newBuilder() to construct.
+ private IOSpecProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private IOSpecProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final IOSpecProto defaultInstance;
+ public static IOSpecProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public IOSpecProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private IOSpecProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ connectedVertexName_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder subBuilder = null;
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ subBuilder = ioDescriptor_.toBuilder();
+ }
+ ioDescriptor_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.PARSER, extensionRegistry);
+ if (subBuilder != null) {
+ subBuilder.mergeFrom(ioDescriptor_);
+ ioDescriptor_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x00000002;
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ physicalEdgeCount_ = input.readInt32();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<IOSpecProto> PARSER =
+ new com.google.protobuf.AbstractParser<IOSpecProto>() {
+ public IOSpecProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new IOSpecProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<IOSpecProto> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // optional string connected_vertex_name = 1;
+ public static final int CONNECTED_VERTEX_NAME_FIELD_NUMBER = 1;
+ private java.lang.Object connectedVertexName_;
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public boolean hasConnectedVertexName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public java.lang.String getConnectedVertexName() {
+ java.lang.Object ref = connectedVertexName_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ connectedVertexName_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getConnectedVertexNameBytes() {
+ java.lang.Object ref = connectedVertexName_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ connectedVertexName_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // optional .EntityDescriptorProto io_descriptor = 2;
+ public static final int IO_DESCRIPTOR_FIELD_NUMBER = 2;
+ private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_;
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public boolean hasIoDescriptor() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
+ return ioDescriptor_;
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
+ return ioDescriptor_;
+ }
+
+ // optional int32 physical_edge_count = 3;
+ public static final int PHYSICAL_EDGE_COUNT_FIELD_NUMBER = 3;
+ private int physicalEdgeCount_;
+ /**
+ * <code>optional int32 physical_edge_count = 3;</code>
+ */
+ public boolean hasPhysicalEdgeCount() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>optional int32 physical_edge_count = 3;</code>
+ */
+ public int getPhysicalEdgeCount() {
+ return physicalEdgeCount_;
+ }
+
+ private void initFields() {
+ connectedVertexName_ = "";
+ ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+ physicalEdgeCount_ = 0;
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getConnectedVertexNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeMessage(2, ioDescriptor_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeInt32(3, physicalEdgeCount_);
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getConnectedVertexNameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(2, ioDescriptor_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(3, physicalEdgeCount_);
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)) {
+ return super.equals(obj);
+ }
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) obj;
+
+ boolean result = true;
+ result = result && (hasConnectedVertexName() == other.hasConnectedVertexName());
+ if (hasConnectedVertexName()) {
+ result = result && getConnectedVertexName()
+ .equals(other.getConnectedVertexName());
+ }
+ result = result && (hasIoDescriptor() == other.hasIoDescriptor());
+ if (hasIoDescriptor()) {
+ result = result && getIoDescriptor()
+ .equals(other.getIoDescriptor());
+ }
+ result = result && (hasPhysicalEdgeCount() == other.hasPhysicalEdgeCount());
+ if (hasPhysicalEdgeCount()) {
+ result = result && (getPhysicalEdgeCount()
+ == other.getPhysicalEdgeCount());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasConnectedVertexName()) {
+ hash = (37 * hash) + CONNECTED_VERTEX_NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getConnectedVertexName().hashCode();
+ }
+ if (hasIoDescriptor()) {
+ hash = (37 * hash) + IO_DESCRIPTOR_FIELD_NUMBER;
+ hash = (53 * hash) + getIoDescriptor().hashCode();
+ }
+ if (hasPhysicalEdgeCount()) {
+ hash = (37 * hash) + PHYSICAL_EDGE_COUNT_FIELD_NUMBER;
+ hash = (53 * hash) + getPhysicalEdgeCount();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code IOSpecProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
+ }
+
+ // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getIoDescriptorFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ connectedVertexName_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ if (ioDescriptorBuilder_ == null) {
+ ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+ } else {
+ ioDescriptorBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ physicalEdgeCount_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto getDefaultInstanceForType() {
+ return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance();
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto build() {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto buildPartial() {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.connectedVertexName_ = connectedVertexName_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ if (ioDescriptorBuilder_ == null) {
+ result.ioDescriptor_ = ioDescriptor_;
+ } else {
+ result.ioDescriptor_ = ioDescriptorBuilder_.build();
+ }
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.physicalEdgeCount_ = physicalEdgeCount_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) {
+ return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other) {
+ if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance()) return this;
+ if (other.hasConnectedVertexName()) {
+ bitField0_ |= 0x00000001;
+ connectedVertexName_ = other.connectedVertexName_;
+ onChanged();
+ }
+ if (other.hasIoDescriptor()) {
+ mergeIoDescriptor(other.getIoDescriptor());
+ }
+ if (other.hasPhysicalEdgeCount()) {
+ setPhysicalEdgeCount(other.getPhysicalEdgeCount());
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // optional string connected_vertex_name = 1;
+ private java.lang.Object connectedVertexName_ = "";
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public boolean hasConnectedVertexName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public java.lang.String getConnectedVertexName() {
+ java.lang.Object ref = connectedVertexName_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ connectedVertexName_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getConnectedVertexNameBytes() {
+ java.lang.Object ref = connectedVertexName_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ connectedVertexName_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public Builder setConnectedVertexName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ connectedVertexName_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public Builder clearConnectedVertexName() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ connectedVertexName_ = getDefaultInstance().getConnectedVertexName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional string connected_vertex_name = 1;</code>
+ */
+ public Builder setConnectedVertexNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ connectedVertexName_ = value;
+ onChanged();
+ return this;
+ }
+
+ // optional .EntityDescriptorProto io_descriptor = 2;
+ private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder> ioDescriptorBuilder_;
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public boolean hasIoDescriptor() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
+ if (ioDescriptorBuilder_ == null) {
+ return ioDescriptor_;
+ } else {
+ return ioDescriptorBuilder_.getMessage();
+ }
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public Builder setIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
+ if (ioDescriptorBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ioDescriptor_ = value;
+ onChanged();
+ } else {
+ ioDescriptorBuilder_.setMessage(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public Builder setIoDescriptor(
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder builderForValue) {
+ if (ioDescriptorBuilder_ == null) {
+ ioDescriptor_ = builderForValue.build();
+ onChanged();
+ } else {
+ ioDescriptorBuilder_.setMessage(builderForValue.build());
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public Builder mergeIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
+ if (ioDescriptorBuilder_ == null) {
+ if (((bitField0_ & 0x00000002) == 0x00000002) &&
+ ioDescriptor_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) {
+ ioDescriptor_ =
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder(ioDescriptor_).mergeFrom(value).buildPartial();
+ } else {
+ ioDescriptor_ = value;
+ }
+ onChanged();
+ } else {
+ ioDescriptorBuilder_.mergeFrom(value);
+ }
+ bitField0_ |= 0x00000002;
+ return this;
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public Builder clearIoDescriptor() {
+ if (ioDescriptorBuilder_ == null) {
+ ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
+ onChanged();
+ } else {
+ ioDescriptorBuilder_.clear();
+ }
+ bitField0_ = (bitField0_ & ~0x00000002);
+ return this;
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder getIoDescriptorBuilder() {
+ bitField0_ |= 0x00000002;
+ onChanged();
+ return getIoDescriptorFieldBuilder().getBuilder();
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
+ if (ioDescriptorBuilder_ != null) {
+ return ioDescriptorBuilder_.getMessageOrBuilder();
+ } else {
+ return ioDescriptor_;
+ }
+ }
+ /**
+ * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
+ */
+ private com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder>
+ getIoDescriptorFieldBuilder() {
+ if (ioDescriptorBuilder_ == null) {
+ ioDescriptorBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+ org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder>(
+ ioDescriptor_,
+ getParentForChildren(),
+ isClean());
+ ioDescriptor_ = null;
+ }
+ return ioDescriptorBuilder_;
+ }
+
+ // optional int32 physical_edge_count = 3;
+ private int physicalEdgeCount_ ;
+ /**
+ * <code>optional int32 physical_edge_cou
<TRUNCATED>
[10/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
HIVE-12934. Refactor llap module structure to allow for a usable client. (Siddharth Seth, reviewed by Sergey Shelukhin)
(cherry picked from commit 4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9886414b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9886414b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9886414b
Branch: refs/heads/branch-2.0
Commit: 9886414b3dd973c1630b84634f0296adac7a193b
Parents: 29a988f
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 1 09:45:09 2016 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Feb 1 09:46:18 2016 -0800
----------------------------------------------------------------------
llap-client/pom.xml | 87 +-
.../hive/llap/security/LlapTokenIdentifier.java | 82 -
.../hive/llap/security/LlapTokenProvider.java | 27 -
.../hive/llap/tez/LlapProtocolClientProxy.java | 509 +
.../tez/TestLlapDaemonProtocolClientProxy.java | 144 +
llap-common/pom.xml | 235 +
.../daemon/rpc/LlapDaemonProtocolProtos.java | 14553 +++++++++++++++++
.../org/apache/hadoop/hive/llap/LlapNodeId.java | 86 +
.../impl/LlapManagementProtocolClientImpl.java | 82 +
.../hive/llap/impl/LlapProtocolClientImpl.java | 125 +
.../llap/protocol/LlapManagementProtocolPB.java | 26 +
.../llap/protocol/LlapProtocolBlockingPB.java | 30 +
.../protocol/LlapTaskUmbilicalProtocol.java | 42 +
.../hive/llap/security/LlapTokenIdentifier.java | 82 +
.../hive/llap/security/LlapTokenProvider.java | 27 +
.../hive/llap/security/LlapTokenSelector.java | 53 +
.../apache/hadoop/hive/llap/tez/Converters.java | 265 +
.../src/protobuf/LlapDaemonProtocol.proto | 148 +
.../hive/llap/testhelpers/ControlledClock.java | 43 +
llap-server/pom.xml | 199 +-
.../daemon/rpc/LlapDaemonProtocolProtos.java | 14553 -----------------
.../org/apache/hadoop/hive/llap/LlapNodeId.java | 86 -
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 8 +
.../daemon/LlapDaemonProtocolBlockingPB.java | 28 -
.../LlapManagementProtocolBlockingPB.java | 24 -
.../hive/llap/daemon/impl/LlapDaemon.java | 4 +-
.../impl/LlapDaemonProtocolClientImpl.java | 125 -
.../impl/LlapDaemonProtocolServerImpl.java | 251 -
.../impl/LlapManagementProtocolClientImpl.java | 82 -
.../daemon/impl/LlapProtocolServerImpl.java | 251 +
.../llap/daemon/impl/QueryFragmentInfo.java | 13 +-
.../llap/daemon/impl/TaskRunnerCallable.java | 2 +-
.../protocol/LlapTaskUmbilicalProtocol.java | 42 -
.../llap/security/LlapDaemonPolicyProvider.java | 8 +-
.../hive/llap/security/LlapSecurityHelper.java | 2 +-
.../llap/security/LlapServerSecurityInfo.java | 10 +-
.../hive/llap/security/LlapTokenSelector.java | 53 -
.../hadoop/hive/llap/tezplugins/Converters.java | 265 -
.../llap/tezplugins/LlapContainerLauncher.java | 43 -
.../LlapDaemonProtocolClientProxy.java | 509 -
.../llap/tezplugins/LlapTaskCommunicator.java | 755 -
.../tezplugins/LlapUmbilicalPolicyProvider.java | 42 -
.../tezplugins/helpers/SourceStateTracker.java | 296 -
.../apache/tez/dag/app/rm/ContainerFactory.java | 51 -
.../dag/app/rm/LlapTaskSchedulerService.java | 1512 --
.../src/protobuf/LlapDaemonProtocol.proto | 148 -
.../impl/TestLlapDaemonProtocolServerImpl.java | 11 +-
.../TestLlapDaemonProtocolClientProxy.java | 143 -
.../tezplugins/TestLlapTaskCommunicator.java | 100 -
.../app/rm/TestLlapTaskSchedulerService.java | 685 -
llap-tez/pom.xml | 200 +
.../hive/llap/tezplugins/ContainerFactory.java | 51 +
.../llap/tezplugins/LlapContainerLauncher.java | 43 +
.../llap/tezplugins/LlapTaskCommunicator.java | 757 +
.../tezplugins/LlapTaskSchedulerService.java | 1512 ++
.../hive/llap/tezplugins/LlapTezUtils.java | 29 +
.../tezplugins/LlapUmbilicalPolicyProvider.java | 42 +
.../tezplugins/helpers/SourceStateTracker.java | 289 +
.../tezplugins/TestLlapTaskCommunicator.java | 100 +
.../TestLlapTaskSchedulerService.java | 684 +
pom.xml | 4 +-
ql/pom.xml | 54 +
.../hive/ql/exec/tez/TezSessionState.java | 58 +-
63 files changed, 20631 insertions(+), 20139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/pom.xml
----------------------------------------------------------------------
diff --git a/llap-client/pom.xml b/llap-client/pom.xml
index 3c85180..a0eeb44 100644
--- a/llap-client/pom.xml
+++ b/llap-client/pom.xml
@@ -32,7 +32,7 @@
</properties>
<dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifectId -->
+ <!-- dependencies are always listed in sorted order by groupId, artifactId -->
<!-- intra-project -->
<dependency>
<groupId>org.apache.hive</groupId>
@@ -41,41 +41,26 @@
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
+ <artifactId>hive-llap-common</artifactId>
<version>${project.version}</version>
</dependency>
<!-- inter-project -->
<dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>${commons-codec.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${commons-lang.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- <version>${libthrift.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
@@ -96,15 +81,15 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>${mockito-all.version}</version>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-servlet</artifactId>
- <version>${jersey.version}</version>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -113,32 +98,18 @@
<version>${hadoop.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commmons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
-
<build>
<sourceDirectory>${basedir}/src/java</sourceDirectory>
<testSourceDirectory>${basedir}/src/test</testSourceDirectory>
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
deleted file mode 100644
index f0bb495..0000000
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-
-/** For now, a LLAP token gives access to any LLAP server. */
-public class LlapTokenIdentifier extends AbstractDelegationTokenIdentifier {
- private static final String KIND = "LLAP_TOKEN";
- public static final Text KIND_NAME = new Text(KIND);
-
- public LlapTokenIdentifier() {
- super();
- }
-
- public LlapTokenIdentifier(Text owner, Text renewer, Text realUser) {
- super(owner, renewer, realUser);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- // Nothing right now.
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- // Nothing right now.
- }
-
- @Override
- public Text getKind() {
- return KIND_NAME;
- }
-
- @Override
- public int hashCode() {
- return -1;
- }
-
- @Override
- public boolean equals(Object other) {
- return (other != null) && other.getClass().isAssignableFrom(this.getClass());
- }
-
- @Override
- public String toString() {
- return KIND;
- }
-
- @InterfaceAudience.Private
- public static class Renewer extends Token.TrivialRenewer {
- @Override
- protected Text getKind() {
- return KIND_NAME;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
deleted file mode 100644
index 2e99a28..0000000
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenProvider.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import java.io.IOException;
-
-import org.apache.hadoop.security.token.Token;
-
-public interface LlapTokenProvider {
- Token<LlapTokenIdentifier> getDelegationToken() throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
new file mode 100644
index 0000000..5b0674a
--- /dev/null
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
@@ -0,0 +1,509 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import javax.net.SocketFactory;
+
+import java.io.IOException;
+import java.security.PrivilegedAction;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.Message;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.impl.LlapProtocolClientImpl;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapProtocolClientProxy extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LlapProtocolClientProxy.class);
+
+ private final ConcurrentMap<String, LlapProtocolBlockingPB> hostProxies;
+
+ private final RequestManager requestManager;
+ private final RetryPolicy retryPolicy;
+ private final SocketFactory socketFactory;
+
+ private final ListeningExecutorService requestManagerExecutor;
+ private volatile ListenableFuture<Void> requestManagerFuture;
+ private final Token<LlapTokenIdentifier> llapToken;
+
+ public LlapProtocolClientProxy(
+ int numThreads, Configuration conf, Token<LlapTokenIdentifier> llapToken) {
+ super(LlapProtocolClientProxy.class.getSimpleName());
+ this.hostProxies = new ConcurrentHashMap<>();
+ this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
+ this.llapToken = llapToken;
+
+ long connectionTimeout = HiveConf.getTimeVar(conf,
+ ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ long retrySleep = HiveConf.getTimeVar(conf,
+ ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MS,
+ TimeUnit.MILLISECONDS);
+ this.retryPolicy = RetryPolicies.retryUpToMaximumTimeWithFixedSleep(
+ connectionTimeout, retrySleep, TimeUnit.MILLISECONDS);
+
+ this.requestManager = new RequestManager(numThreads);
+ ExecutorService localExecutor = Executors.newFixedThreadPool(1,
+ new ThreadFactoryBuilder().setNameFormat("RequestManagerExecutor").build());
+ this.requestManagerExecutor = MoreExecutors.listeningDecorator(localExecutor);
+
+ LOG.info("Setting up taskCommunicator with" +
+ "numThreads=" + numThreads +
+ "retryTime(millis)=" + connectionTimeout +
+ "retrySleep(millis)=" + retrySleep);
+ }
+
+ @Override
+ public void serviceStart() {
+ requestManagerFuture = requestManagerExecutor.submit(requestManager);
+ Futures.addCallback(requestManagerFuture, new FutureCallback<Void>() {
+ @Override
+ public void onSuccess(Void result) {
+ LOG.info("RequestManager shutdown");
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.warn("RequestManager shutdown with error", t);
+ }
+ });
+ }
+
+ @Override
+ public void serviceStop() {
+ if (requestManagerFuture != null) {
+ requestManager.shutdown();
+ requestManagerFuture.cancel(true);
+ }
+ requestManagerExecutor.shutdown();
+ }
+
+ public void sendSubmitWork(SubmitWorkRequestProto request, String host, int port,
+ final ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
+ LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+ requestManager.queueRequest(new SubmitWorkCallable(nodeId, request, callback));
+ }
+
+ public void sendSourceStateUpdate(final SourceStateUpdatedRequestProto request, final String host,
+ final int port,
+ final ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
+ LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+ requestManager.queueRequest(
+ new SendSourceStateUpdateCallable(nodeId, request, callback));
+ }
+
+ public void sendQueryComplete(final QueryCompleteRequestProto request, final String host,
+ final int port,
+ final ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
+ LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+ requestManager.queueRequest(new SendQueryCompleteCallable(nodeId, request, callback));
+ }
+
+ public void sendTerminateFragment(final TerminateFragmentRequestProto request, final String host,
+ final int port,
+ final ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
+ LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+ requestManager.queueRequest(new SendTerminateFragmentCallable(nodeId, request, callback));
+ }
+
+ @VisibleForTesting
+ static class RequestManager implements Callable<Void> {
+
+ private final Lock lock = new ReentrantLock();
+ private final AtomicBoolean isShutdown = new AtomicBoolean(false);
+ private final Condition queueCondition = lock.newCondition();
+ private final AtomicBoolean shouldRun = new AtomicBoolean(false);
+
+ private final int maxConcurrentRequestsPerNode = 1;
+ private final ListeningExecutorService executor;
+
+
+ // Tracks new additions via add, while the loop is processing existing ones.
+ private final LinkedList<CallableRequest> newRequestList = new LinkedList<>();
+
+ // Tracks existing requests which are cycled through.
+ private final LinkedList<CallableRequest> pendingRequests = new LinkedList<>();
+
+ // Tracks requests executing per node
+ private final ConcurrentMap<LlapNodeId, AtomicInteger> runningRequests = new ConcurrentHashMap<>();
+
+ // Tracks completed requests pre node
+ private final LinkedList<LlapNodeId> completedNodes = new LinkedList<>();
+
+ public RequestManager(int numThreads) {
+ ExecutorService localExecutor = Executors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder().setNameFormat("TaskCommunicator #%2d").build());
+ executor = MoreExecutors.listeningDecorator(localExecutor);
+ }
+
+
+ @VisibleForTesting
+ Set<LlapNodeId> currentLoopDisabledNodes = new HashSet<>();
+ @VisibleForTesting
+ List<CallableRequest> currentLoopSkippedRequests = new LinkedList<>();
+ @Override
+ public Void call() {
+ // Caches disabled nodes for quicker lookups and ensures a request on a node which was skipped
+ // does not go out of order.
+ while (!isShutdown.get()) {
+ lock.lock();
+ try {
+ while (!shouldRun.get()) {
+ queueCondition.await();
+ break; // Break out and try executing.
+ }
+ boolean shouldBreak = process();
+ if (shouldBreak) {
+ break;
+ }
+ } catch (InterruptedException e) {
+ if (isShutdown.get()) {
+ break;
+ } else {
+ LOG.warn("RunLoop interrupted without being shutdown first");
+ throw new RuntimeException(e);
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+ LOG.info("CallScheduler loop exiting");
+ return null;
+ }
+
+ /* Add a new request to be executed */
+ public void queueRequest(CallableRequest request) {
+ synchronized (newRequestList) {
+ newRequestList.add(request);
+ shouldRun.set(true);
+ }
+ notifyRunLoop();
+ }
+
+ /* Indicates a request has completed on a node */
+ public void requestFinished(LlapNodeId nodeId) {
+ synchronized (completedNodes) {
+ completedNodes.add(nodeId);
+ shouldRun.set(true);
+ }
+ notifyRunLoop();
+ }
+
+ public void shutdown() {
+ if (!isShutdown.getAndSet(true)) {
+ executor.shutdownNow();
+ notifyRunLoop();
+ }
+ }
+
+ @VisibleForTesting
+ void submitToExecutor(CallableRequest request, LlapNodeId nodeId) {
+ ListenableFuture<SourceStateUpdatedResponseProto> future =
+ executor.submit(request);
+ Futures.addCallback(future, new ResponseCallback(request.getCallback(), nodeId, this));
+ }
+
+ @VisibleForTesting
+ boolean process() {
+ if (isShutdown.get()) {
+ return true;
+ }
+ currentLoopDisabledNodes.clear();
+ currentLoopSkippedRequests.clear();
+
+ // Set to false to block the next loop. This must be called before draining the lists,
+ // otherwise an add/completion after draining the lists but before setting it to false,
+ // will not trigger a run. May cause one unnecessary run if an add comes in before drain.
+ // drain list. add request (setTrue). setFalse needs to be avoided.
+ shouldRun.compareAndSet(true, false);
+ // Drain any calls which may have come in during the last execution of the loop.
+ drainNewRequestList(); // Locks newRequestList
+ drainCompletedNodes(); // Locks completedNodes
+
+
+ Iterator<CallableRequest> iterator = pendingRequests.iterator();
+ while (iterator.hasNext()) {
+ CallableRequest request = iterator.next();
+ iterator.remove();
+ LlapNodeId nodeId = request.getNodeId();
+ if (canRunForNode(nodeId, currentLoopDisabledNodes)) {
+ submitToExecutor(request, nodeId);
+ } else {
+ currentLoopDisabledNodes.add(nodeId);
+ currentLoopSkippedRequests.add(request);
+ }
+ }
+ // Tried scheduling everything that could be scheduled in this loop.
+ pendingRequests.addAll(0, currentLoopSkippedRequests);
+ return false;
+ }
+
+ private void drainNewRequestList() {
+ synchronized (newRequestList) {
+ if (!newRequestList.isEmpty()) {
+ pendingRequests.addAll(newRequestList);
+ newRequestList.clear();
+ }
+ }
+ }
+
+ private void drainCompletedNodes() {
+ synchronized (completedNodes) {
+ if (!completedNodes.isEmpty()) {
+ for (LlapNodeId nodeId : completedNodes) {
+ runningRequests.get(nodeId).decrementAndGet();
+ }
+ }
+ completedNodes.clear();
+ }
+ }
+
+ private boolean canRunForNode(LlapNodeId nodeId, Set<LlapNodeId> currentRunDisabledNodes) {
+ if (currentRunDisabledNodes.contains(nodeId)) {
+ return false;
+ } else {
+ AtomicInteger count = runningRequests.get(nodeId);
+ if (count == null) {
+ count = new AtomicInteger(0);
+ AtomicInteger old = runningRequests.putIfAbsent(nodeId, count);
+ count = old != null ? old : count;
+ }
+ if (count.incrementAndGet() <= maxConcurrentRequestsPerNode) {
+ return true;
+ } else {
+ count.decrementAndGet();
+ return false;
+ }
+ }
+ }
+
+ private void notifyRunLoop() {
+ lock.lock();
+ try {
+ queueCondition.signal();
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+
+
+ private static final class ResponseCallback<TYPE extends Message>
+ implements FutureCallback<TYPE> {
+
+ private final ExecuteRequestCallback<TYPE> callback;
+ private final LlapNodeId nodeId;
+ private final RequestManager requestManager;
+
+ public ResponseCallback(ExecuteRequestCallback<TYPE> callback, LlapNodeId nodeId,
+ RequestManager requestManager) {
+ this.callback = callback;
+ this.nodeId = nodeId;
+ this.requestManager = requestManager;
+ }
+
+ @Override
+ public void onSuccess(TYPE result) {
+ try {
+ callback.setResponse(result);
+ } finally {
+ requestManager.requestFinished(nodeId);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ try {
+ callback.indicateError(t);
+ } finally {
+ requestManager.requestFinished(nodeId);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ static abstract class CallableRequest<REQUEST extends Message, RESPONSE extends Message>
+ implements Callable {
+
+ final LlapNodeId nodeId;
+ final ExecuteRequestCallback<RESPONSE> callback;
+ final REQUEST request;
+
+
+ protected CallableRequest(LlapNodeId nodeId, REQUEST request, ExecuteRequestCallback<RESPONSE> callback) {
+ this.nodeId = nodeId;
+ this.request = request;
+ this.callback = callback;
+ }
+
+ public LlapNodeId getNodeId() {
+ return nodeId;
+ }
+
+ public ExecuteRequestCallback<RESPONSE> getCallback() {
+ return callback;
+ }
+
+ public abstract RESPONSE call() throws Exception;
+ }
+
+ private class SubmitWorkCallable extends CallableRequest<SubmitWorkRequestProto, SubmitWorkResponseProto> {
+
+ protected SubmitWorkCallable(LlapNodeId nodeId,
+ SubmitWorkRequestProto submitWorkRequestProto,
+ ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
+ super(nodeId, submitWorkRequestProto, callback);
+ }
+
+ @Override
+ public SubmitWorkResponseProto call() throws Exception {
+ return getProxy(nodeId).submitWork(null, request);
+ }
+ }
+
+ private class SendSourceStateUpdateCallable
+ extends CallableRequest<SourceStateUpdatedRequestProto, SourceStateUpdatedResponseProto> {
+
+ public SendSourceStateUpdateCallable(LlapNodeId nodeId,
+ SourceStateUpdatedRequestProto request,
+ ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
+ super(nodeId, request, callback);
+ }
+
+ @Override
+ public SourceStateUpdatedResponseProto call() throws Exception {
+ return getProxy(nodeId).sourceStateUpdated(null, request);
+ }
+ }
+
+ private class SendQueryCompleteCallable
+ extends CallableRequest<QueryCompleteRequestProto, QueryCompleteResponseProto> {
+
+ protected SendQueryCompleteCallable(LlapNodeId nodeId,
+ QueryCompleteRequestProto queryCompleteRequestProto,
+ ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
+ super(nodeId, queryCompleteRequestProto, callback);
+ }
+
+ @Override
+ public QueryCompleteResponseProto call() throws Exception {
+ return getProxy(nodeId).queryComplete(null, request);
+ }
+ }
+
+ private class SendTerminateFragmentCallable
+ extends CallableRequest<TerminateFragmentRequestProto, TerminateFragmentResponseProto> {
+
+ protected SendTerminateFragmentCallable(LlapNodeId nodeId,
+ TerminateFragmentRequestProto terminateFragmentRequestProto,
+ ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
+ super(nodeId, terminateFragmentRequestProto, callback);
+ }
+
+ @Override
+ public TerminateFragmentResponseProto call() throws Exception {
+ return getProxy(nodeId).terminateFragment(null, request);
+ }
+ }
+
+ public interface ExecuteRequestCallback<T extends Message> {
+ void setResponse(T response);
+ void indicateError(Throwable t);
+ }
+
+ private LlapProtocolBlockingPB getProxy(final LlapNodeId nodeId) {
+ String hostId = getHostIdentifier(nodeId.getHostname(), nodeId.getPort());
+
+ LlapProtocolBlockingPB proxy = hostProxies.get(hostId);
+ if (proxy == null) {
+ if (llapToken == null) {
+ proxy = new LlapProtocolClientImpl(getConfig(), nodeId.getHostname(),
+ nodeId.getPort(), retryPolicy, socketFactory);
+ } else {
+ UserGroupInformation ugi;
+ try {
+ ugi = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Token<LlapTokenIdentifier> nodeToken = new Token<LlapTokenIdentifier>(llapToken);
+ SecurityUtil.setTokenService(nodeToken, NetUtils.createSocketAddrForHost(
+ nodeId.getHostname(), nodeId.getPort()));
+ ugi.addToken(nodeToken);
+ proxy = ugi.doAs(new PrivilegedAction<LlapProtocolBlockingPB>() {
+ @Override
+ public LlapProtocolBlockingPB run() {
+ return new LlapProtocolClientImpl(getConfig(), nodeId.getHostname(),
+ nodeId.getPort(), retryPolicy, socketFactory);
+ }
+ });
+ }
+
+ LlapProtocolBlockingPB proxyOld = hostProxies.putIfAbsent(hostId, proxy);
+ if (proxyOld != null) {
+ // TODO Shutdown the new proxy.
+ proxy = proxyOld;
+ }
+ }
+ return proxy;
+ }
+
+ private String getHostIdentifier(String hostname, int port) {
+ return hostname + ":" + port;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java b/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java
new file mode 100644
index 0000000..850b67a
--- /dev/null
+++ b/llap-client/src/test/org/apache/hadoop/hive/llap/tez/TestLlapDaemonProtocolClientProxy.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tez;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.protobuf.Message;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestLlapDaemonProtocolClientProxy {
+
+ @Test (timeout = 5000)
+ public void testMultipleNodes() {
+ RequestManagerForTest requestManager = new RequestManagerForTest(1);
+
+ LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
+ LlapNodeId nodeId2 = LlapNodeId.getInstance("host2", 1025);
+
+ Message mockMessage = mock(Message.class);
+ LlapProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
+ LlapProtocolClientProxy.ExecuteRequestCallback.class);
+
+ // Request two messages
+ requestManager.queueRequest(
+ new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
+ requestManager.queueRequest(
+ new CallableRequestForTest(nodeId2, mockMessage, mockExecuteRequestCallback));
+
+ // Should go through in a single process call
+ requestManager.process();
+ assertEquals(2, requestManager.numSubmissionsCounters);
+ assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+ assertNotNull(requestManager.numInvocationsPerNode.get(nodeId2));
+ Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+ Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId2).getValue().intValue());
+ assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+ assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+ assertEquals(0, requestManager.currentLoopDisabledNodes.size());
+ }
+
+ @Test(timeout = 5000)
+ public void testSingleInvocationPerNode() {
+ RequestManagerForTest requestManager = new RequestManagerForTest(1);
+
+ LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
+
+ Message mockMessage = mock(Message.class);
+ LlapProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
+ LlapProtocolClientProxy.ExecuteRequestCallback.class);
+
+ // First request for host.
+ requestManager.queueRequest(
+ new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
+ requestManager.process();
+ assertEquals(1, requestManager.numSubmissionsCounters);
+ assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+ Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+ assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+
+ // Second request for host. Single invocation since the last has not completed.
+ requestManager.queueRequest(
+ new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
+ requestManager.process();
+ assertEquals(1, requestManager.numSubmissionsCounters);
+ assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+ Assert.assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+ assertEquals(1, requestManager.currentLoopSkippedRequests.size());
+ assertEquals(1, requestManager.currentLoopDisabledNodes.size());
+ assertTrue(requestManager.currentLoopDisabledNodes.contains(nodeId1));
+
+ // Complete first request. Second pending request should go through.
+ requestManager.requestFinished(nodeId1);
+ requestManager.process();
+ assertEquals(2, requestManager.numSubmissionsCounters);
+ assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
+ Assert.assertEquals(2, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
+ assertEquals(0, requestManager.currentLoopSkippedRequests.size());
+ assertEquals(0, requestManager.currentLoopDisabledNodes.size());
+ assertFalse(requestManager.currentLoopDisabledNodes.contains(nodeId1));
+ }
+
+
+ static class RequestManagerForTest extends LlapProtocolClientProxy.RequestManager {
+
+ int numSubmissionsCounters = 0;
+ private Map<LlapNodeId, MutableInt> numInvocationsPerNode = new HashMap<>();
+
+ public RequestManagerForTest(int numThreads) {
+ super(numThreads);
+ }
+
+ protected void submitToExecutor(LlapProtocolClientProxy.CallableRequest request, LlapNodeId nodeId) {
+ numSubmissionsCounters++;
+ MutableInt nodeCount = numInvocationsPerNode.get(nodeId);
+ if (nodeCount == null) {
+ nodeCount = new MutableInt(0);
+ numInvocationsPerNode.put(nodeId, nodeCount);
+ }
+ nodeCount.increment();
+ }
+
+ void reset() {
+ numSubmissionsCounters = 0;
+ numInvocationsPerNode.clear();
+ }
+
+ }
+
+ static class CallableRequestForTest extends LlapProtocolClientProxy.CallableRequest<Message, Message> {
+
+ protected CallableRequestForTest(LlapNodeId nodeId, Message message,
+ LlapProtocolClientProxy.ExecuteRequestCallback<Message> callback) {
+ super(nodeId, message, callback);
+ }
+
+ @Override
+ public Message call() throws Exception {
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-common/pom.xml
----------------------------------------------------------------------
diff --git a/llap-common/pom.xml b/llap-common/pom.xml
new file mode 100644
index 0000000..5343479
--- /dev/null
+++ b/llap-common/pom.xml
@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>hive-llap-common</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Llap Common</name>
+
+ <properties>
+ <hive.path.to.root>..</hive.path.to.root>
+ </properties>
+
+ <dependencies>
+ <!-- dependencies are always listed in sorted order by groupId, artifactId -->
+ <!-- intra-project -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- inter-project -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-internals</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- test inter-project -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>protobuf</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-protobuf-sources</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <target>
+ <property name="protobuf.src.dir" location="${basedir}/src/protobuf"/>
+ <property name="protobuf.build.dir" location="${basedir}/src/gen/protobuf/gen-java"/>
+ <echo>Building LLAP Protobuf</echo>
+ <mkdir dir="${protobuf.build.dir}"/>
+ <exec executable="protoc" failonerror="true">
+ <arg value="--java_out=${protobuf.build.dir}"/>
+ <arg value="-I=${protobuf.src.dir}"/>
+ <arg value="${protobuf.src.dir}/LlapDaemonProtocol.proto"/>
+ </exec>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+
+ <build>
+ <sourceDirectory>${basedir}/src/java</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>*.py</exclude>
+ <exclude>*.pyc</exclude>
+ </excludes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/gen/protobuf/gen-java</source>
+ <source>src/gen/thrift/gen-javabean</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
[03/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java b/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
deleted file mode 100644
index 4c1cbb3..0000000
--- a/llap-server/src/test/org/apache/tez/dag/app/rm/TestLlapTaskSchedulerService.java
+++ /dev/null
@@ -1,685 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
-import org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.app.ControlledClock;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestLlapTaskSchedulerService {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestLlapTaskSchedulerService.class);
-
- private static final String HOST1 = "host1";
- private static final String HOST2 = "host2";
- private static final String HOST3 = "host3";
-
- @Test (timeout = 5000)
- public void testSimpleLocalAllocation() throws IOException, InterruptedException {
-
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
-
- try {
- Priority priority1 = Priority.newInstance(1);
- String[] hosts1 = new String[]{HOST1};
-
- Object task1 = new Object();
- Object clientCookie1 = new Object();
-
- tsWrapper.controlScheduler(true);
- tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
-
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
-
- verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
- // TODO Verify this is on host1.
- assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
- } finally {
- tsWrapper.shutdown();
- }
- }
-
- @Test (timeout = 5000)
- public void testSimpleNoLocalityAllocation() throws IOException, InterruptedException {
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper();
-
- try {
- Priority priority1 = Priority.newInstance(1);
-
- Object task1 = new Object();
- Object clientCookie1 = new Object();
- tsWrapper.controlScheduler(true);
- tsWrapper.allocateTask(task1, null, priority1, clientCookie1);
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1), any(Container.class));
- assertEquals(1, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
- } finally {
- tsWrapper.shutdown();
- }
- }
-
-
- @Test(timeout=5000)
- public void testPreemption() throws InterruptedException, IOException {
-
- Priority priority1 = Priority.newInstance(1);
- Priority priority2 = Priority.newInstance(2);
- String [] hosts = new String[] {HOST1};
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1);
- try {
-
- Object task1 = "task1";
- Object clientCookie1 = "cookie1";
- Object task2 = "task2";
- Object clientCookie2 = "cookie2";
- Object task3 = "task3";
- Object clientCookie3 = "cookie3";
- Object task4 = "task4";
- Object clientCookie4 = "cookie4";
-
- tsWrapper.controlScheduler(true);
- tsWrapper.allocateTask(task1, hosts, priority2, clientCookie1);
- tsWrapper.allocateTask(task2, hosts, priority2, clientCookie2);
- tsWrapper.allocateTask(task3, hosts, priority2, clientCookie3);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback, times(2)).taskAllocated(any(Object.class),
- any(Object.class), any(Container.class));
- assertEquals(2, tsWrapper.ts.dagStats.numLocalAllocations);
- assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
-
- reset(tsWrapper.mockAppCallback);
-
- tsWrapper.allocateTask(task4, hosts, priority1, clientCookie4);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
-
-
- tsWrapper.deallocateTask(task2, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
-
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
- eq(clientCookie4), any(Container.class));
-
- } finally {
- tsWrapper.shutdown();
- }
-
- }
-
- @Test(timeout=5000)
- public void testNodeDisabled() throws IOException, InterruptedException {
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(10000l);
- try {
- Priority priority1 = Priority.newInstance(1);
- String[] hosts1 = new String[]{HOST1};
- Object task1 = new Object();
- Object clientCookie1 = new Object();
- tsWrapper.controlScheduler(true);
- tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 1) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback).taskAllocated(eq(task1), eq(clientCookie1),
- any(Container.class));
- assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
- assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
- assertEquals(0, tsWrapper.ts.dagStats.numNonLocalAllocations);
- assertEquals(1, tsWrapper.ts.dagStats.numTotalAllocations);
-
- tsWrapper.resetAppCallback();
-
- tsWrapper.clock.setTime(10000l);
- tsWrapper.rejectExecution(task1);
-
- // Verify that the node is blacklisted
- assertEquals(1, tsWrapper.ts.dagStats.numRejectedTasks);
- assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
- LlapTaskSchedulerService.NodeInfo disabledNodeInfo = tsWrapper.ts.disabledNodesQueue.peek();
- assertNotNull(disabledNodeInfo);
- assertEquals(HOST1, disabledNodeInfo.serviceInstance.getHost());
- assertEquals((10000l), disabledNodeInfo.getDelay(TimeUnit.MILLISECONDS));
- assertEquals((10000l + 10000l), disabledNodeInfo.expireTimeMillis);
-
- Object task2 = new Object();
- Object clientCookie2 = new Object();
- tsWrapper.allocateTask(task2, hosts1, priority1, clientCookie2);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback).taskAllocated(eq(task2), eq(clientCookie2), any(Container.class));
- assertEquals(1, tsWrapper.ts.dagStats.numLocalAllocations);
- assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
- assertEquals(1, tsWrapper.ts.dagStats.numNonLocalAllocations);
- assertEquals(2, tsWrapper.ts.dagStats.numTotalAllocations);
-
- } finally {
- tsWrapper.shutdown();
- }
- }
-
- @Test(timeout=5000)
- public void testNodeReEnabled() throws InterruptedException, IOException {
- // Based on actual timing.
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(1000l);
- try {
- Priority priority1 = Priority.newInstance(1);
- String[] hosts1 = new String[]{HOST1};
- String[] hosts2 = new String[]{HOST2};
- String[] hosts3 = new String[]{HOST3};
-
- Object task1 = new Object();
- Object clientCookie1 = new Object();
- Object task2 = new Object();
- Object clientCookie2 = new Object();
- Object task3 = new Object();
- Object clientCookie3 = new Object();
-
- tsWrapper.controlScheduler(true);
- tsWrapper.allocateTask(task1, hosts1, priority1, clientCookie1);
- tsWrapper.allocateTask(task2, hosts2, priority1, clientCookie2);
- tsWrapper.allocateTask(task3, hosts3, priority1, clientCookie3);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), any(Container.class));
- assertEquals(3, tsWrapper.ts.dagStats.numLocalAllocations);
- assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
- assertEquals(3, tsWrapper.ts.dagStats.numTotalAllocations);
-
- tsWrapper.resetAppCallback();
-
- tsWrapper.rejectExecution(task1);
- tsWrapper.rejectExecution(task2);
- tsWrapper.rejectExecution(task3);
-
- // Verify that the node is blacklisted
- assertEquals(3, tsWrapper.ts.dagStats.numRejectedTasks);
- assertEquals(3, tsWrapper.ts.instanceToNodeMap.size());
- assertEquals(3, tsWrapper.ts.disabledNodesQueue.size());
-
-
- Object task4 = new Object();
- Object clientCookie4 = new Object();
- Object task5 = new Object();
- Object clientCookie5 = new Object();
- Object task6 = new Object();
- Object clientCookie6 = new Object();
- tsWrapper.allocateTask(task4, hosts1, priority1, clientCookie4);
- tsWrapper.allocateTask(task5, hosts2, priority1, clientCookie5);
- tsWrapper.allocateTask(task6, hosts3, priority1, clientCookie6);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 6) {
- break;
- }
- }
-
- ArgumentCaptor<Container> argumentCaptor = ArgumentCaptor.forClass(Container.class);
- verify(tsWrapper.mockAppCallback, times(3)).taskAllocated(any(Object.class), any(Object.class), argumentCaptor.capture());
-
- // which affects the locality matching
- assertEquals(0, tsWrapper.ts.dagStats.numAllocationsNoLocalityRequest);
- assertEquals(6, tsWrapper.ts.dagStats.numTotalAllocations);
-
- } finally {
- tsWrapper.shutdown();
- }
- }
-
- @Test (timeout = 5000)
- public void testForceLocalityTest1() throws IOException, InterruptedException {
- // 2 hosts. 2 per host. 5 requests at the same priority.
- // First 3 on host1, Next at host2, Last with no host.
- // Third request on host1 should not be allocated immediately.
- forceLocalityTest1(true);
-
- }
-
- @Test (timeout = 5000)
- public void testNoForceLocalityCounterTest1() throws IOException, InterruptedException {
- // 2 hosts. 2 per host. 5 requests at the same priority.
- // First 3 on host1, Next at host2, Last with no host.
- // Third should allocate on host2, 4th on host2, 5th will wait.
-
- forceLocalityTest1(false);
- }
-
- private void forceLocalityTest1(boolean forceLocality) throws IOException, InterruptedException {
- Priority priority1 = Priority.newInstance(1);
-
- String[] hosts = new String[] {HOST1, HOST2};
-
- String[] hostsH1 = new String[] {HOST1};
- String[] hostsH2 = new String[] {HOST2};
-
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, (forceLocality ? -1l : 0l));
-
- try {
- Object task1 = "task1";
- Object clientCookie1 = "cookie1";
- Object task2 = "task2";
- Object clientCookie2 = "cookie2";
- Object task3 = "task3";
- Object clientCookie3 = "cookie3";
- Object task4 = "task4";
- Object clientCookie4 = "cookie4";
- Object task5 = "task5";
- Object clientCookie5 = "cookie5";
-
- tsWrapper.controlScheduler(true);
- //H1 - should allocate
- tsWrapper.allocateTask(task1, hostsH1, priority1, clientCookie1);
- //H1 - should allocate
- tsWrapper.allocateTask(task2, hostsH1, priority1, clientCookie2);
- //H1 - no capacity if force, should allocate otherwise
- tsWrapper.allocateTask(task3, hostsH1, priority1, clientCookie3);
- //H2 - should allocate
- tsWrapper.allocateTask(task4, hostsH2, priority1, clientCookie4);
- //No location - should allocate if force, no capacity otherwise
- tsWrapper.allocateTask(task5, null, priority1, clientCookie5);
-
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 4) {
- break;
- }
- }
-
- // Verify no preemption requests - since everything is at the same priority
- verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
- ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
- verify(tsWrapper.mockAppCallback, times(4)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
- assertEquals(4, argumentCaptor.getAllValues().size());
- assertEquals(task1, argumentCaptor.getAllValues().get(0));
- assertEquals(task2, argumentCaptor.getAllValues().get(1));
- if (forceLocality) {
- // task3 not allocated
- assertEquals(task4, argumentCaptor.getAllValues().get(2));
- assertEquals(task5, argumentCaptor.getAllValues().get(3));
- } else {
- assertEquals(task3, argumentCaptor.getAllValues().get(2));
- assertEquals(task4, argumentCaptor.getAllValues().get(3));
- }
-
- //Complete one task on host1.
- tsWrapper.deallocateTask(task1, true, null);
-
- reset(tsWrapper.mockAppCallback);
-
- // Try scheduling again.
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 5) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
- argumentCaptor = ArgumentCaptor.forClass(Object.class);
- verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
- assertEquals(1, argumentCaptor.getAllValues().size());
- if (forceLocality) {
- assertEquals(task3, argumentCaptor.getAllValues().get(0));
- } else {
- assertEquals(task5, argumentCaptor.getAllValues().get(0));
- }
-
- } finally {
- tsWrapper.shutdown();
- }
- }
-
- @Test(timeout = 5000)
- public void testForcedLocalityUnknownHost() throws IOException, InterruptedException {
- Priority priority1 = Priority.newInstance(1);
-
- String[] hostsKnown = new String[]{HOST1};
- String[] hostsUnknown = new String[]{HOST2};
-
- TestTaskSchedulerServiceWrapper tsWrapper =
- new TestTaskSchedulerServiceWrapper(2000, hostsKnown, 1, 1, -1l);
- try {
- Object task1 = "task1";
- Object clientCookie1 = "cookie1";
-
- Object task2 = "task2";
- Object clientCookie2 = "cookie2";
-
- tsWrapper.controlScheduler(true);
- // Should allocate since H2 is not known.
- tsWrapper.allocateTask(task1, hostsUnknown, priority1, clientCookie1);
- tsWrapper.allocateTask(task2, hostsKnown, priority1, clientCookie2);
-
-
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 2) {
- break;
- }
- }
-
- ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
- verify(tsWrapper.mockAppCallback, times(2))
- .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
- assertEquals(2, argumentCaptor.getAllValues().size());
- assertEquals(task1, argumentCaptor.getAllValues().get(0));
- assertEquals(task2, argumentCaptor.getAllValues().get(1));
-
-
- } finally {
- tsWrapper.shutdown();
- }
- }
-
-
- @Test(timeout = 5000)
- public void testForcedLocalityPreemption() throws IOException, InterruptedException {
- Priority priority1 = Priority.newInstance(1);
- Priority priority2 = Priority.newInstance(2);
- String [] hosts = new String[] {HOST1, HOST2};
-
- String [] hostsH1 = new String[] {HOST1};
- String [] hostsH2 = new String[] {HOST2};
-
- TestTaskSchedulerServiceWrapper tsWrapper = new TestTaskSchedulerServiceWrapper(2000, hosts, 1, 1, -1l);
-
- // Fill up host1 with p2 tasks.
- // Leave host2 empty
- // Try running p1 task on host1 - should preempt
-
- try {
- Object task1 = "task1";
- Object clientCookie1 = "cookie1";
- Object task2 = "task2";
- Object clientCookie2 = "cookie2";
- Object task3 = "task3";
- Object clientCookie3 = "cookie3";
- Object task4 = "task4";
- Object clientCookie4 = "cookie4";
-
- tsWrapper.controlScheduler(true);
- tsWrapper.allocateTask(task1, hostsH1, priority2, clientCookie1);
- tsWrapper.allocateTask(task2, hostsH1, priority2, clientCookie2);
- // This request at a lower priority should not affect anything.
- tsWrapper.allocateTask(task3, hostsH1, priority2, clientCookie3);
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numLocalAllocations == 2) {
- break;
- }
- }
-
- verify(tsWrapper.mockAppCallback, never()).preemptContainer(any(ContainerId.class));
- ArgumentCaptor<Object> argumentCaptor = ArgumentCaptor.forClass(Object.class);
- verify(tsWrapper.mockAppCallback, times(2))
- .taskAllocated(argumentCaptor.capture(), any(Object.class), any(Container.class));
- assertEquals(2, argumentCaptor.getAllValues().size());
- assertEquals(task1, argumentCaptor.getAllValues().get(0));
- assertEquals(task2, argumentCaptor.getAllValues().get(1));
-
- reset(tsWrapper.mockAppCallback);
- // Allocate t4 at higher priority. t3 should not be allocated,
- // and a preemption should be attempted on host1, despite host2 having available capacity
- tsWrapper.allocateTask(task4, hostsH1, priority1, clientCookie4);
-
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numPreemptedTasks == 1) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback).preemptContainer(any(ContainerId.class));
-
- tsWrapper.deallocateTask(task1, false, TaskAttemptEndReason.INTERNAL_PREEMPTION);
-
- while (true) {
- tsWrapper.signalSchedulerRun();
- tsWrapper.awaitSchedulerRun();
- if (tsWrapper.ts.dagStats.numTotalAllocations == 3) {
- break;
- }
- }
- verify(tsWrapper.mockAppCallback, times(1)).taskAllocated(eq(task4),
- eq(clientCookie4), any(Container.class));
-
- } finally {
- tsWrapper.shutdown();
- }
- }
-
- private static class TestTaskSchedulerServiceWrapper {
- static final Resource resource = Resource.newInstance(1024, 1);
- Configuration conf;
- TaskSchedulerContext mockAppCallback = mock(TaskSchedulerContext.class);
- ControlledClock clock = new ControlledClock(new SystemClock());
- ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1000, 1), 1);
- LlapTaskSchedulerServiceForTest ts;
-
- TestTaskSchedulerServiceWrapper() throws IOException, InterruptedException {
- this(2000l);
- }
-
- TestTaskSchedulerServiceWrapper(long disableTimeoutMillis) throws IOException,
- InterruptedException {
- this(disableTimeoutMillis, new String[]{HOST1, HOST2, HOST3}, 4,
- ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.defaultIntVal);
- }
-
- TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize) throws
- IOException, InterruptedException {
- this(disableTimeoutMillis, hosts, numExecutors, waitQueueSize, 0l);
- }
-
- TestTaskSchedulerServiceWrapper(long disableTimeoutMillis, String[] hosts, int numExecutors, int waitQueueSize, long localityDelayMs) throws
- IOException, InterruptedException {
- conf = new Configuration();
- conf.setStrings(ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, hosts);
- conf.setInt(ConfVars.LLAP_DAEMON_NUM_EXECUTORS.varname, numExecutors);
- conf.setInt(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname, waitQueueSize);
- conf.set(ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS.varname,
- disableTimeoutMillis + "ms");
- conf.setBoolean(LlapFixedRegistryImpl.FIXED_REGISTRY_RESOLVE_HOST_NAMES, false);
- conf.setLong(ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY.varname, localityDelayMs);
-
- doReturn(appAttemptId).when(mockAppCallback).getApplicationAttemptId();
- doReturn(11111l).when(mockAppCallback).getCustomClusterIdentifier();
- UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
- doReturn(userPayload).when(mockAppCallback).getInitialUserPayload();
-
- ts = new LlapTaskSchedulerServiceForTest(mockAppCallback, clock);
-
- controlScheduler(true);
- ts.initialize();
- ts.start();
- // One scheduler pass from the nodes that are added at startup
- signalSchedulerRun();
- controlScheduler(false);
- awaitSchedulerRun();
- }
-
- void controlScheduler(boolean val) {
- ts.forTestsetControlScheduling(val);
- }
-
- void signalSchedulerRun() throws InterruptedException {
- ts.forTestSignalSchedulingRun();
- }
-
- void awaitSchedulerRun() throws InterruptedException {
- ts.forTestAwaitSchedulingRun();
- }
- void resetAppCallback() {
- reset(mockAppCallback);
- }
-
- void shutdown() {
- ts.shutdown();
- }
-
- void allocateTask(Object task, String[] hosts, Priority priority, Object clientCookie) {
- ts.allocateTask(task, resource, hosts, null, priority, null, clientCookie);
- }
-
- void deallocateTask(Object task, boolean succeeded, TaskAttemptEndReason endReason) {
- ts.deallocateTask(task, succeeded, endReason, null);
- }
-
- void rejectExecution(Object task) {
- ts.deallocateTask(task, false, TaskAttemptEndReason.EXECUTOR_BUSY, null);
- }
- }
-
- private static class LlapTaskSchedulerServiceForTest extends LlapTaskSchedulerService {
-
- private AtomicBoolean controlScheduling = new AtomicBoolean(false);
- private final Lock testLock = new ReentrantLock();
- private final Condition schedulingCompleteCondition = testLock.newCondition();
- private boolean schedulingComplete = false;
- private final Condition triggerSchedulingCondition = testLock.newCondition();
- private boolean schedulingTriggered = false;
- private final AtomicInteger numSchedulerRuns = new AtomicInteger(0);
-
-
- public LlapTaskSchedulerServiceForTest(
- TaskSchedulerContext appClient, Clock clock) {
- super(appClient, clock);
- }
-
- @Override
- protected void schedulePendingTasks() {
- testLock.lock();
- try {
- if (controlScheduling.get()) {
- while (!schedulingTriggered) {
- try {
- triggerSchedulingCondition.await();
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
- numSchedulerRuns.incrementAndGet();
- super.schedulePendingTasks();
- schedulingTriggered = false;
- schedulingComplete = true;
- schedulingCompleteCondition.signal();
- } finally {
- testLock.unlock();
- }
- }
-
- // Enable or disable test scheduling control.
- void forTestsetControlScheduling(boolean control) {
- this.controlScheduling.set(control);
- }
-
- void forTestSignalSchedulingRun() throws InterruptedException {
- testLock.lock();
- try {
- schedulingTriggered = true;
- triggerSchedulingCondition.signal();
- } finally {
- testLock.unlock();
- }
- }
-
- void forTestAwaitSchedulingRun() throws InterruptedException {
- testLock.lock();
- try {
- while (!schedulingComplete) {
- schedulingCompleteCondition.await();
- }
- schedulingComplete = false;
- } finally {
- testLock.unlock();
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/pom.xml
----------------------------------------------------------------------
diff --git a/llap-tez/pom.xml b/llap-tez/pom.xml
new file mode 100644
index 0000000..ce020da
--- /dev/null
+++ b/llap-tez/pom.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>hive-llap-tez</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Llap Tez</name>
+
+ <properties>
+ <hive.path.to.root>..</hive.path.to.root>
+ </properties>
+
+ <dependencies>
+ <!-- dependencies are always listed in sorted order by groupId, artifactId -->
+ <!-- intra-project -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- inter-project -->
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-internals</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ <version>${tez.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- test intra-project -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- test inter-project -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commmons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <sourceDirectory>${basedir}/src/java</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/gen/protobuf/gen-java</source>
+ <source>src/gen/thrift/gen-javabean</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java
new file mode 100644
index 0000000..a314391
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/ContainerFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+class ContainerFactory {
+ final ApplicationAttemptId customAppAttemptId;
+ AtomicLong nextId;
+
+ public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) {
+ this.nextId = new AtomicLong(1);
+ ApplicationId appId =
+ ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId());
+ this.customAppAttemptId =
+ ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId());
+ }
+
+ public Container createContainer(Resource capability, Priority priority, String hostname,
+ int port) {
+ ContainerId containerId =
+ ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement());
+ NodeId nodeId = NodeId.newInstance(hostname, port);
+ String nodeHttpAddress = "hostname:0"; // TODO: include UI ports
+
+ Container container =
+ Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null);
+
+ return container;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
new file mode 100644
index 0000000..07703a2
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
+import org.apache.tez.serviceplugins.api.ContainerLauncher;
+import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
+import org.apache.tez.serviceplugins.api.ContainerStopRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapContainerLauncher extends ContainerLauncher {
+ private static final Logger LOG = LoggerFactory.getLogger(LlapContainerLauncher.class);
+
+ public LlapContainerLauncher(ContainerLauncherContext containerLauncherContext) {
+ super(containerLauncherContext);
+ }
+
+ @Override
+ public void launchContainer(ContainerLaunchRequest containerLaunchRequest) {
+ LOG.info("No-op launch for container: " + containerLaunchRequest.getContainerId() +
+ " succeeded on host: " + containerLaunchRequest.getNodeId());
+ getContext().containerLaunched(containerLaunchRequest.getContainerId());
+ }
+
+ @Override
+ public void stopContainer(ContainerStopRequest containerStopRequest) {
+ LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + containerStopRequest);
+ getContext().containerStopRequested(containerStopRequest.getContainerId());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
new file mode 100644
index 0000000..76d095a
--- /dev/null
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -0,0 +1,757 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.tezplugins;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapNodeId;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
+import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
+import org.apache.hadoop.hive.llap.tez.Converters;
+import org.apache.hadoop.hive.llap.tez.LlapProtocolClientProxy;
+import org.apache.hadoop.hive.llap.tezplugins.helpers.SourceStateTracker;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezException;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.event.VertexStateUpdate;
+import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
+import org.apache.tez.dag.records.TezTaskAttemptID;
+import org.apache.tez.runtime.api.impl.TaskSpec;
+import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
+import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
+import org.apache.tez.serviceplugins.api.ContainerEndReason;
+import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
+import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LlapTaskCommunicator.class);
+
+ private static final boolean isInfoEnabled = LOG.isInfoEnabled();
+ private static final boolean isDebugEnabed = LOG.isDebugEnabled();
+
+ private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
+
+ private final ConcurrentMap<QueryIdentifierProto, ByteBuffer> credentialMap;
+
+ // Tracks containerIds and taskAttemptIds, so can be kept independent of the running DAG.
+ // When DAG specific cleanup happens, it'll be better to link this to a DAG though.
+ private final EntityTracker entityTracker = new EntityTracker();
+ private final SourceStateTracker sourceStateTracker;
+ private final Set<LlapNodeId> nodesForQuery = new HashSet<>();
+
+ private LlapProtocolClientProxy communicator;
+ private long deleteDelayOnDagComplete;
+ private final LlapTaskUmbilicalProtocol umbilical;
+ private final Token<LlapTokenIdentifier> token;
+
+ // These two structures track the list of known nodes, and the list of nodes which are sending in keep-alive heartbeats.
+ // Primarily for debugging purposes a.t.m, since there's some unexplained TASK_TIMEOUTS which are currently being observed.
+ private final ConcurrentMap<LlapNodeId, Long> knownNodeMap = new ConcurrentHashMap<>();
+ private final ConcurrentMap<LlapNodeId, PingingNodeInfo> pingedNodeMap = new ConcurrentHashMap<>();
+
+
+ private volatile int currentDagId;
+ private volatile QueryIdentifierProto currentQueryIdentifierProto;
+
+ public LlapTaskCommunicator(
+ TaskCommunicatorContext taskCommunicatorContext) {
+ super(taskCommunicatorContext);
+ Credentials credentials = taskCommunicatorContext.getCredentials();
+ if (credentials != null) {
+ @SuppressWarnings("unchecked")
+ Token<LlapTokenIdentifier> llapToken =
+ (Token<LlapTokenIdentifier>)credentials.getToken(LlapTokenIdentifier.KIND_NAME);
+ this.token = llapToken;
+ } else {
+ this.token = null;
+ }
+ Preconditions.checkState((token != null) == UserGroupInformation.isSecurityEnabled());
+
+ umbilical = new LlapTaskUmbilicalProtocolImpl(getUmbilical());
+ SubmitWorkRequestProto.Builder baseBuilder = SubmitWorkRequestProto.newBuilder();
+
+ // TODO Avoid reading this from the environment
+ baseBuilder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
+ baseBuilder.setApplicationIdString(
+ taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString());
+ baseBuilder
+ .setAppAttemptNumber(taskCommunicatorContext.getApplicationAttemptId().getAttemptId());
+ baseBuilder.setTokenIdentifier(getTokenIdentifier());
+
+ BASE_SUBMIT_WORK_REQUEST = baseBuilder.build();
+
+ credentialMap = new ConcurrentHashMap<>();
+ sourceStateTracker = new SourceStateTracker(getContext(), this);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ super.initialize();
+ Configuration conf = getConf();
+ int numThreads = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS);
+ this.communicator = new LlapProtocolClientProxy(numThreads, conf, token);
+ this.deleteDelayOnDagComplete = HiveConf.getTimeVar(
+ conf, ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS);
+ LOG.info("Running LlapTaskCommunicator with "
+ + "fileCleanupDelay=" + deleteDelayOnDagComplete
+ + ", numCommunicatorThreads=" + numThreads);
+ this.communicator.init(conf);
+ }
+
+ @Override
+ public void start() {
+ super.start();
+ this.communicator.start();
+ }
+
+ @Override
+ public void shutdown() {
+ super.shutdown();
+ if (this.communicator != null) {
+ this.communicator.stop();
+ }
+ }
+
+ @Override
+ protected void startRpcServer() {
+ Configuration conf = getConf();
+ try {
+ JobTokenSecretManager jobTokenSecretManager =
+ new JobTokenSecretManager();
+ jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
+
+ int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
+ TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT);
+ server = new RPC.Builder(conf)
+ .setProtocol(LlapTaskUmbilicalProtocol.class)
+ .setBindAddress("0.0.0.0")
+ .setPort(0)
+ .setInstance(umbilical)
+ .setNumHandlers(numHandlers)
+ .setSecretManager(jobTokenSecretManager).build();
+
+ if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
+ server.refreshServiceAcl(conf, new LlapUmbilicalPolicyProvider());
+ }
+
+ server.start();
+ this.address = NetUtils.getConnectAddress(server);
+ LOG.info(
+ "Started LlapUmbilical: " + umbilical.getClass().getName() + " at address: " + address +
+ " with numHandlers=" + numHandlers);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ }
+
+ @Override
+ public void registerRunningContainer(ContainerId containerId, String hostname, int port) {
+ super.registerRunningContainer(containerId, hostname, port);
+ entityTracker.registerContainer(containerId, hostname, port);
+
+ }
+
+ @Override
+ public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, String diagnostics) {
+ super.registerContainerEnd(containerId, endReason, diagnostics);
+ if (endReason == ContainerEndReason.INTERNAL_PREEMPTION) {
+ LOG.info("Processing containerEnd for container {} caused by internal preemption", containerId);
+ TezTaskAttemptID taskAttemptId = entityTracker.getTaskAttemptIdForContainer(containerId);
+ if (taskAttemptId != null) {
+ sendTaskTerminated(taskAttemptId, true);
+ }
+ }
+ entityTracker.unregisterContainer(containerId);
+ }
+
+
+ @Override
+ public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
+ Map<String, LocalResource> additionalResources,
+ Credentials credentials,
+ boolean credentialsChanged,
+ int priority) {
+ super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
+ credentialsChanged, priority);
+ int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId();
+ if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIdentifier())) {
+ resetCurrentDag(dagId);
+ }
+
+
+ ContainerInfo containerInfo = getContainerInfo(containerId);
+ String host;
+ int port;
+ if (containerInfo != null) {
+ synchronized (containerInfo) {
+ host = containerInfo.host;
+ port = containerInfo.port;
+ }
+ } else {
+ // TODO Handle this properly
+ throw new RuntimeException("ContainerInfo not found for container: " + containerId +
+ ", while trying to launch task: " + taskSpec.getTaskAttemptID());
+ }
+
+ LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
+ registerKnownNode(nodeId);
+ entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port);
+ nodesForQuery.add(nodeId);
+
+ sourceStateTracker.registerTaskForStateUpdates(host, port, taskSpec.getInputs());
+ FragmentRuntimeInfo fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo(
+ taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority);
+ SubmitWorkRequestProto requestProto;
+
+ try {
+ requestProto = constructSubmitWorkRequest(containerId, taskSpec, fragmentRuntimeInfo);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to construct request", e);
+ }
+
+ // Have to register this up front right now. Otherwise, it's possible for the task to start
+ // sending out status/DONE/KILLED/FAILED messages before TAImpl knows how to handle them.
+ getContext()
+ .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
+ communicator.sendSubmitWork(requestProto, host, port,
+ new LlapProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
+ @Override
+ public void setResponse(SubmitWorkResponseProto response) {
+ if (response.hasSubmissionState()) {
+ LlapDaemonProtocolProtos.SubmissionStateProto ss = response.getSubmissionState();
+ if (ss.equals(LlapDaemonProtocolProtos.SubmissionStateProto.REJECTED)) {
+ LOG.info(
+ "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+ containerId + ", Service Busy");
+ getContext().taskKilled(taskSpec.getTaskAttemptID(),
+ TaskAttemptEndReason.EXECUTOR_BUSY, "Service Busy");
+ return;
+ }
+ } else {
+ // TODO: Provide support for reporting errors
+ // This should never happen as server always returns a valid status on success
+ throw new RuntimeException("SubmissionState in response is expected!");
+ }
+ LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ if (t instanceof ServiceException) {
+ ServiceException se = (ServiceException) t;
+ t = se.getCause();
+ }
+ if (t instanceof RemoteException) {
+ RemoteException re = (RemoteException) t;
+ // All others from the remote service cause the task to FAIL.
+ LOG.info(
+ "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+ containerId, t);
+ getContext()
+ .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
+ t.toString());
+ } else {
+ // Exception from the RPC layer - communication failure, consider as KILLED / service down.
+ if (t instanceof IOException) {
+ LOG.info(
+ "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+ containerId + ", Communication Error");
+ getContext().taskKilled(taskSpec.getTaskAttemptID(),
+ TaskAttemptEndReason.COMMUNICATION_ERROR, "Communication Error");
+ } else {
+ // Anything else is a FAIL.
+ LOG.info(
+ "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
+ containerId, t);
+ getContext()
+ .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
+ t.getMessage());
+ }
+ }
+ }
+ });
+ }
+
+ @Override
+ public void unregisterRunningTaskAttempt(final TezTaskAttemptID taskAttemptId,
+ TaskAttemptEndReason endReason,
+ String diagnostics) {
+ super.unregisterRunningTaskAttempt(taskAttemptId, endReason, diagnostics);
+
+ if (endReason == TaskAttemptEndReason.INTERNAL_PREEMPTION) {
+ LOG.info("Processing taskEnd for task {} caused by internal preemption", taskAttemptId);
+ sendTaskTerminated(taskAttemptId, false);
+ }
+ entityTracker.unregisterTaskAttempt(taskAttemptId);
+ // This will also be invoked for tasks which have been KILLED / rejected by the daemon.
+ // Informing the daemon becomes necessary once the LlapScheduler supports preemption
+ // and/or starts attempting to kill tasks which may be running on a node.
+ }
+
+ private void sendTaskTerminated(final TezTaskAttemptID taskAttemptId,
+ boolean invokedByContainerEnd) {
+ LOG.info(
+ "DBG: Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}",
+ taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd");
+ LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId);
+ // NodeId can be null if the task gets unregistered due to failure / being killed by the daemon itself
+ if (nodeId != null) {
+ TerminateFragmentRequestProto request =
+ TerminateFragmentRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifierProto)
+ .setFragmentIdentifierString(taskAttemptId.toString()).build();
+ communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(),
+ new LlapProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() {
+ @Override
+ public void setResponse(TerminateFragmentResponseProto response) {
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ LOG.warn("Failed to send terminate fragment request for {}",
+ taskAttemptId.toString());
+ }
+ });
+ } else {
+ LOG.info(
+ "Not sending terminate request for fragment {} since it's node is not known. Already unregistered",
+ taskAttemptId.toString());
+ }
+ }
+
+
+
+
+ @Override
+ public void dagComplete(final int dagIdentifier) {
+ QueryCompleteRequestProto request = QueryCompleteRequestProto.newBuilder()
+ .setQueryIdentifier(constructQueryIdentifierProto(dagIdentifier))
+ .setDeleteDelay(deleteDelayOnDagComplete).build();
+ for (final LlapNodeId llapNodeId : nodesForQuery) {
+ LOG.info("Sending dagComplete message for {}, to {}", dagIdentifier, llapNodeId);
+ communicator.sendQueryComplete(request, llapNodeId.getHostname(), llapNodeId.getPort(),
+ new LlapProtocolClientProxy.ExecuteRequestCallback<LlapDaemonProtocolProtos.QueryCompleteResponseProto>() {
+ @Override
+ public void setResponse(LlapDaemonProtocolProtos.QueryCompleteResponseProto response) {
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ LOG.warn("Failed to indicate dag complete dagId={} to node {}", dagIdentifier, llapNodeId);
+ }
+ });
+ }
+
+ nodesForQuery.clear();
+ // TODO Ideally move some of the other cleanup code from resetCurrentDag over here
+ }
+
+ @Override
+ public void onVertexStateUpdated(VertexStateUpdate vertexStateUpdate) {
+ // Delegate updates over to the source state tracker.
+ sourceStateTracker
+ .sourceStateUpdated(vertexStateUpdate.getVertexName(), vertexStateUpdate.getVertexState());
+ }
+
+ public void sendStateUpdate(final String host, final int port,
+ final SourceStateUpdatedRequestProto request) {
+ communicator.sendSourceStateUpdate(request, host, port,
+ new LlapProtocolClientProxy.ExecuteRequestCallback<SourceStateUpdatedResponseProto>() {
+ @Override
+ public void setResponse(SourceStateUpdatedResponseProto response) {
+ }
+
+ @Override
+ public void indicateError(Throwable t) {
+ // TODO HIVE-10280.
+ // Ideally, this should be retried for a while, after which the node should be marked as failed.
+ // Considering tasks are supposed to run fast. Failing the task immediately may be a good option.
+ LOG.error(
+ "Failed to send state update to node: " + host + ":" + port + ", StateUpdate=" +
+ request, t);
+ }
+ });
+ }
+
+
+ private static class PingingNodeInfo {
+ final AtomicLong logTimestamp;
+ final AtomicInteger pingCount;
+
+ PingingNodeInfo(long currentTs) {
+ logTimestamp = new AtomicLong(currentTs);
+ pingCount = new AtomicInteger(1);
+ }
+ }
+
+ public void registerKnownNode(LlapNodeId nodeId) {
+ Long old = knownNodeMap.putIfAbsent(nodeId,
+ TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS));
+ if (old == null) {
+ if (isInfoEnabled) {
+ LOG.info("Added new known node: {}", nodeId);
+ }
+ }
+ }
+
+ public void registerPingingNode(LlapNodeId nodeId) {
+ long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
+ PingingNodeInfo ni = new PingingNodeInfo(currentTs);
+ PingingNodeInfo old = pingedNodeMap.put(nodeId, ni);
+ if (old == null) {
+ if (isInfoEnabled) {
+ LOG.info("Added new pinging node: [{}]", nodeId);
+ }
+ } else {
+ old.pingCount.incrementAndGet();
+ }
+ // The node should always be known by this point. Log occasionally if it is not known.
+ if (!knownNodeMap.containsKey(nodeId)) {
+ if (old == null) {
+ // First time this is seen. Log it.
+ LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, ni.pingCount.get());
+ } else {
+ // Pinged before. Log only occasionally.
+ if (currentTs > old.logTimestamp.get() + 5000l) { // 5 seconds elapsed. Log again.
+ LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, old.pingCount.get());
+ old.logTimestamp.set(currentTs);
+ }
+ }
+
+ }
+ }
+
+
+ private final AtomicLong nodeNotFoundLogTime = new AtomicLong(0);
+
+ void nodePinged(String hostname, int port) {
+ LlapNodeId nodeId = LlapNodeId.getInstance(hostname, port);
+ registerPingingNode(nodeId);
+ BiMap<ContainerId, TezTaskAttemptID> biMap =
+ entityTracker.getContainerAttemptMapForNode(nodeId);
+ if (biMap != null) {
+ synchronized (biMap) {
+ for (Map.Entry<ContainerId, TezTaskAttemptID> entry : biMap.entrySet()) {
+ getContext().taskAlive(entry.getValue());
+ getContext().containerAlive(entry.getKey());
+ }
+ }
+ } else {
+ long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
+ if (currentTs > nodeNotFoundLogTime.get() + 5000l) {
+ LOG.warn("Received ping from node without any registered tasks or containers: " + hostname +
+ ":" + port +
+ ". Could be caused by pre-emption by the AM," +
+ " or a mismatched hostname. Enable debug logging for mismatched host names");
+ nodeNotFoundLogTime.set(currentTs);
+ }
+ }
+ }
+
+ private void resetCurrentDag(int newDagId) {
+ // Working on the assumption that a single DAG runs at a time per AM.
+ currentQueryIdentifierProto = constructQueryIdentifierProto(newDagId);
+ sourceStateTracker.resetState(newDagId);
+ nodesForQuery.clear();
+ LOG.info("CurrentDagId set to: " + newDagId + ", name=" + getContext().getCurrentDagName());
+ // TODO Is it possible for heartbeats to come in from lost tasks - those should be told to die, which
+ // is likely already happening.
+ }
+
+ private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId,
+ TaskSpec taskSpec,
+ FragmentRuntimeInfo fragmentRuntimeInfo) throws
+ IOException {
+ SubmitWorkRequestProto.Builder builder =
+ SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
+ builder.setContainerIdString(containerId.toString());
+ builder.setAmHost(getAddress().getHostName());
+ builder.setAmPort(getAddress().getPort());
+ Credentials taskCredentials = new Credentials();
+ // Credentials can change across DAGs. Ideally construct only once per DAG.
+ taskCredentials.addAll(getContext().getCredentials());
+
+ Preconditions.checkState(currentQueryIdentifierProto.getDagIdentifier() ==
+ taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId());
+ ByteBuffer credentialsBinary = credentialMap.get(currentQueryIdentifierProto);
+ if (credentialsBinary == null) {
+ credentialsBinary = serializeCredentials(getContext().getCredentials());
+ credentialMap.putIfAbsent(currentQueryIdentifierProto, credentialsBinary.duplicate());
+ } else {
+ credentialsBinary = credentialsBinary.duplicate();
+ }
+ builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
+ builder.setFragmentSpec(Converters.convertTaskSpecToProto(taskSpec));
+ builder.setFragmentRuntimeInfo(fragmentRuntimeInfo);
+ return builder.build();
+ }
+
+ private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
+ Credentials containerCredentials = new Credentials();
+ containerCredentials.addAll(credentials);
+ DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
+ containerCredentials.writeTokenStorageToStream(containerTokens_dob);
+ return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
+ }
+
+
+
+ protected class LlapTaskUmbilicalProtocolImpl implements LlapTaskUmbilicalProtocol {
+
+ private final TezTaskUmbilicalProtocol tezUmbilical;
+
+ public LlapTaskUmbilicalProtocolImpl(TezTaskUmbilicalProtocol tezUmbilical) {
+ this.tezUmbilical = tezUmbilical;
+ }
+
+ @Override
+ public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
+ return tezUmbilical.canCommit(taskid);
+ }
+
+ @Override
+ public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
+ TezException {
+ return tezUmbilical.heartbeat(request);
+ }
+
+ @Override
+ public void nodeHeartbeat(Text hostname, int port) throws IOException {
+ nodePinged(hostname.toString(), port);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received heartbeat from [" + hostname + ":" + port +"]");
+ }
+ }
+
+ @Override
+ public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException {
+ // TODO Unregister the task for state updates, which could in turn unregister the node.
+ getContext().taskKilled(taskAttemptId,
+ TaskAttemptEndReason.EXTERNAL_PREEMPTION, "Attempt preempted");
+ entityTracker.unregisterTaskAttempt(taskAttemptId);
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return versionID;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return ProtocolSignature.getProtocolSignature(this, protocol,
+ clientVersion, clientMethodsHash);
+ }
+ }
+
+ /**
+ * Track the association between known containers and taskAttempts, along with the nodes they are assigned to.
+ */
+ @VisibleForTesting
+ static final class EntityTracker {
+ @VisibleForTesting
+ final ConcurrentMap<TezTaskAttemptID, LlapNodeId> attemptToNodeMap = new ConcurrentHashMap<>();
+ @VisibleForTesting
+ final ConcurrentMap<ContainerId, LlapNodeId> containerToNodeMap = new ConcurrentHashMap<>();
+ @VisibleForTesting
+ final ConcurrentMap<LlapNodeId, BiMap<ContainerId, TezTaskAttemptID>> nodeMap = new ConcurrentHashMap<>();
+
+ void registerTaskAttempt(ContainerId containerId, TezTaskAttemptID taskAttemptId, String host, int port) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Registering " + containerId + ", " + taskAttemptId + " for node: " + host + ":" + port);
+ }
+ LlapNodeId llapNodeId = LlapNodeId.getInstance(host, port);
+ attemptToNodeMap.putIfAbsent(taskAttemptId, llapNodeId);
+
+ registerContainer(containerId, host, port);
+
+ // nodeMap registration.
+ BiMap<ContainerId, TezTaskAttemptID> tmpMap = HashBiMap.create();
+ BiMap<ContainerId, TezTaskAttemptID> old = nodeMap.putIfAbsent(llapNodeId, tmpMap);
+ BiMap<ContainerId, TezTaskAttemptID> usedInstance;
+ usedInstance = old == null ? tmpMap : old;
+ synchronized(usedInstance) {
+ usedInstance.put(containerId, taskAttemptId);
+ }
+ // Make sure to put the instance back again, in case it was removed as part of a
+ // containerEnd/taskEnd invocation.
+ nodeMap.putIfAbsent(llapNodeId, usedInstance);
+ }
+
+ void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
+ LlapNodeId llapNodeId = attemptToNodeMap.remove(attemptId);
+ if (llapNodeId == null) {
+ // Possible since either container / task can be unregistered.
+ return;
+ }
+
+ BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
+ ContainerId matched = null;
+ if (bMap != null) {
+ synchronized(bMap) {
+ matched = bMap.inverse().remove(attemptId);
+ }
+ if (bMap.isEmpty()) {
+ nodeMap.remove(llapNodeId);
+ }
+ }
+
+ // Remove the container mapping
+ if (matched != null) {
+ containerToNodeMap.remove(matched);
+ }
+
+ }
+
+ void registerContainer(ContainerId containerId, String hostname, int port) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Registering " + containerId + " for node: " + hostname + ":" + port);
+ }
+ containerToNodeMap.putIfAbsent(containerId, LlapNodeId.getInstance(hostname, port));
+ // nodeMap registration is not required, since there's no taskId association.
+ }
+
+ LlapNodeId getNodeIdForContainer(ContainerId containerId) {
+ return containerToNodeMap.get(containerId);
+ }
+
+ LlapNodeId getNodeIdForTaskAttempt(TezTaskAttemptID taskAttemptId) {
+ return attemptToNodeMap.get(taskAttemptId);
+ }
+
+ ContainerId getContainerIdForAttempt(TezTaskAttemptID taskAttemptId) {
+ LlapNodeId llapNodeId = getNodeIdForTaskAttempt(taskAttemptId);
+ if (llapNodeId != null) {
+ BiMap<TezTaskAttemptID, ContainerId> bMap = nodeMap.get(llapNodeId).inverse();
+ if (bMap != null) {
+ synchronized (bMap) {
+ return bMap.get(taskAttemptId);
+ }
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ TezTaskAttemptID getTaskAttemptIdForContainer(ContainerId containerId) {
+ LlapNodeId llapNodeId = getNodeIdForContainer(containerId);
+ if (llapNodeId != null) {
+ BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
+ if (bMap != null) {
+ synchronized (bMap) {
+ return bMap.get(containerId);
+ }
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+ void unregisterContainer(ContainerId containerId) {
+ LlapNodeId llapNodeId = containerToNodeMap.remove(containerId);
+ if (llapNodeId == null) {
+ // Possible since either container / task can be unregistered.
+ return;
+ }
+
+ BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
+ TezTaskAttemptID matched = null;
+ if (bMap != null) {
+ synchronized(bMap) {
+ matched = bMap.remove(containerId);
+ }
+ if (bMap.isEmpty()) {
+ nodeMap.remove(llapNodeId);
+ }
+ }
+
+ // Remove the container mapping
+ if (matched != null) {
+ attemptToNodeMap.remove(matched);
+ }
+ }
+
+ /**
+ * Return a {@link BiMap} containing container->taskAttemptId mapping for the host specified.
+ * </p>
+ * <p/>
+ * This method return the internal structure used by the EntityTracker. Users must synchronize
+ * on the structure to ensure correct usage.
+ *
+ * @param llapNodeId
+ * @return
+ */
+ BiMap<ContainerId, TezTaskAttemptID> getContainerAttemptMapForNode(LlapNodeId llapNodeId) {
+ BiMap<ContainerId, TezTaskAttemptID> biMap = nodeMap.get(llapNodeId);
+ return biMap;
+ }
+
+ }
+
+ private QueryIdentifierProto constructQueryIdentifierProto(int dagIdentifier) {
+ return QueryIdentifierProto.newBuilder()
+ .setAppIdentifier(getContext().getCurrentAppIdentifier()).setDagIdentifier(dagIdentifier)
+ .build();
+ }
+}
[07/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
----------------------------------------------------------------------
diff --git a/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java b/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
deleted file mode 100644
index 4ab7b32..0000000
--- a/llap-server/src/gen/protobuf/gen-java/org/apache/hadoop/hive/llap/daemon/rpc/LlapDaemonProtocolProtos.java
+++ /dev/null
@@ -1,14553 +0,0 @@
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: LlapDaemonProtocol.proto
-
-package org.apache.hadoop.hive.llap.daemon.rpc;
-
-public final class LlapDaemonProtocolProtos {
- private LlapDaemonProtocolProtos() {}
- public static void registerAllExtensions(
- com.google.protobuf.ExtensionRegistry registry) {
- }
- /**
- * Protobuf enum {@code SourceStateProto}
- */
- public enum SourceStateProto
- implements com.google.protobuf.ProtocolMessageEnum {
- /**
- * <code>S_SUCCEEDED = 1;</code>
- */
- S_SUCCEEDED(0, 1),
- /**
- * <code>S_RUNNING = 2;</code>
- */
- S_RUNNING(1, 2),
- ;
-
- /**
- * <code>S_SUCCEEDED = 1;</code>
- */
- public static final int S_SUCCEEDED_VALUE = 1;
- /**
- * <code>S_RUNNING = 2;</code>
- */
- public static final int S_RUNNING_VALUE = 2;
-
-
- public final int getNumber() { return value; }
-
- public static SourceStateProto valueOf(int value) {
- switch (value) {
- case 1: return S_SUCCEEDED;
- case 2: return S_RUNNING;
- default: return null;
- }
- }
-
- public static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
- internalGetValueMap() {
- return internalValueMap;
- }
- private static com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>
- internalValueMap =
- new com.google.protobuf.Internal.EnumLiteMap<SourceStateProto>() {
- public SourceStateProto findValueByNumber(int number) {
- return SourceStateProto.valueOf(number);
- }
- };
-
- public final com.google.protobuf.Descriptors.EnumValueDescriptor
- getValueDescriptor() {
- return getDescriptor().getValues().get(index);
- }
- public final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptorForType() {
- return getDescriptor();
- }
- public static final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(0);
- }
-
- private static final SourceStateProto[] VALUES = values();
-
- public static SourceStateProto valueOf(
- com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
- if (desc.getType() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "EnumValueDescriptor is not for this type.");
- }
- return VALUES[desc.getIndex()];
- }
-
- private final int index;
- private final int value;
-
- private SourceStateProto(int index, int value) {
- this.index = index;
- this.value = value;
- }
-
- // @@protoc_insertion_point(enum_scope:SourceStateProto)
- }
-
- /**
- * Protobuf enum {@code SubmissionStateProto}
- */
- public enum SubmissionStateProto
- implements com.google.protobuf.ProtocolMessageEnum {
- /**
- * <code>ACCEPTED = 1;</code>
- */
- ACCEPTED(0, 1),
- /**
- * <code>REJECTED = 2;</code>
- */
- REJECTED(1, 2),
- /**
- * <code>EVICTED_OTHER = 3;</code>
- */
- EVICTED_OTHER(2, 3),
- ;
-
- /**
- * <code>ACCEPTED = 1;</code>
- */
- public static final int ACCEPTED_VALUE = 1;
- /**
- * <code>REJECTED = 2;</code>
- */
- public static final int REJECTED_VALUE = 2;
- /**
- * <code>EVICTED_OTHER = 3;</code>
- */
- public static final int EVICTED_OTHER_VALUE = 3;
-
-
- public final int getNumber() { return value; }
-
- public static SubmissionStateProto valueOf(int value) {
- switch (value) {
- case 1: return ACCEPTED;
- case 2: return REJECTED;
- case 3: return EVICTED_OTHER;
- default: return null;
- }
- }
-
- public static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
- internalGetValueMap() {
- return internalValueMap;
- }
- private static com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>
- internalValueMap =
- new com.google.protobuf.Internal.EnumLiteMap<SubmissionStateProto>() {
- public SubmissionStateProto findValueByNumber(int number) {
- return SubmissionStateProto.valueOf(number);
- }
- };
-
- public final com.google.protobuf.Descriptors.EnumValueDescriptor
- getValueDescriptor() {
- return getDescriptor().getValues().get(index);
- }
- public final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptorForType() {
- return getDescriptor();
- }
- public static final com.google.protobuf.Descriptors.EnumDescriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.getDescriptor().getEnumTypes().get(1);
- }
-
- private static final SubmissionStateProto[] VALUES = values();
-
- public static SubmissionStateProto valueOf(
- com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
- if (desc.getType() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "EnumValueDescriptor is not for this type.");
- }
- return VALUES[desc.getIndex()];
- }
-
- private final int index;
- private final int value;
-
- private SubmissionStateProto(int index, int value) {
- this.index = index;
- this.value = value;
- }
-
- // @@protoc_insertion_point(enum_scope:SubmissionStateProto)
- }
-
- public interface UserPayloadProtoOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional bytes user_payload = 1;
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- boolean hasUserPayload();
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- com.google.protobuf.ByteString getUserPayload();
-
- // optional int32 version = 2;
- /**
- * <code>optional int32 version = 2;</code>
- */
- boolean hasVersion();
- /**
- * <code>optional int32 version = 2;</code>
- */
- int getVersion();
- }
- /**
- * Protobuf type {@code UserPayloadProto}
- */
- public static final class UserPayloadProto extends
- com.google.protobuf.GeneratedMessage
- implements UserPayloadProtoOrBuilder {
- // Use UserPayloadProto.newBuilder() to construct.
- private UserPayloadProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private UserPayloadProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final UserPayloadProto defaultInstance;
- public static UserPayloadProto getDefaultInstance() {
- return defaultInstance;
- }
-
- public UserPayloadProto getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private UserPayloadProto(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- userPayload_ = input.readBytes();
- break;
- }
- case 16: {
- bitField0_ |= 0x00000002;
- version_ = input.readInt32();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
- }
-
- public static com.google.protobuf.Parser<UserPayloadProto> PARSER =
- new com.google.protobuf.AbstractParser<UserPayloadProto>() {
- public UserPayloadProto parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new UserPayloadProto(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<UserPayloadProto> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // optional bytes user_payload = 1;
- public static final int USER_PAYLOAD_FIELD_NUMBER = 1;
- private com.google.protobuf.ByteString userPayload_;
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- public boolean hasUserPayload() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- public com.google.protobuf.ByteString getUserPayload() {
- return userPayload_;
- }
-
- // optional int32 version = 2;
- public static final int VERSION_FIELD_NUMBER = 2;
- private int version_;
- /**
- * <code>optional int32 version = 2;</code>
- */
- public boolean hasVersion() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional int32 version = 2;</code>
- */
- public int getVersion() {
- return version_;
- }
-
- private void initFields() {
- userPayload_ = com.google.protobuf.ByteString.EMPTY;
- version_ = 0;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, userPayload_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeInt32(2, version_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, userPayload_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(2, version_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)) {
- return super.equals(obj);
- }
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) obj;
-
- boolean result = true;
- result = result && (hasUserPayload() == other.hasUserPayload());
- if (hasUserPayload()) {
- result = result && getUserPayload()
- .equals(other.getUserPayload());
- }
- result = result && (hasVersion() == other.hasVersion());
- if (hasVersion()) {
- result = result && (getVersion()
- == other.getVersion());
- }
- result = result &&
- getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasUserPayload()) {
- hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
- hash = (53 * hash) + getUserPayload().hashCode();
- }
- if (hasVersion()) {
- hash = (37 * hash) + VERSION_FIELD_NUMBER;
- hash = (53 * hash) + getVersion();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code UserPayloadProto}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- userPayload_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000001);
- version_ = 0;
- bitField0_ = (bitField0_ & ~0x00000002);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_UserPayloadProto_descriptor;
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getDefaultInstanceForType() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto build() {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto buildPartial() {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.userPayload_ = userPayload_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.version_ = version_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) {
- return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto other) {
- if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) return this;
- if (other.hasUserPayload()) {
- setUserPayload(other.getUserPayload());
- }
- if (other.hasVersion()) {
- setVersion(other.getVersion());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // optional bytes user_payload = 1;
- private com.google.protobuf.ByteString userPayload_ = com.google.protobuf.ByteString.EMPTY;
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- public boolean hasUserPayload() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- public com.google.protobuf.ByteString getUserPayload() {
- return userPayload_;
- }
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- public Builder setUserPayload(com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- userPayload_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional bytes user_payload = 1;</code>
- */
- public Builder clearUserPayload() {
- bitField0_ = (bitField0_ & ~0x00000001);
- userPayload_ = getDefaultInstance().getUserPayload();
- onChanged();
- return this;
- }
-
- // optional int32 version = 2;
- private int version_ ;
- /**
- * <code>optional int32 version = 2;</code>
- */
- public boolean hasVersion() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional int32 version = 2;</code>
- */
- public int getVersion() {
- return version_;
- }
- /**
- * <code>optional int32 version = 2;</code>
- */
- public Builder setVersion(int value) {
- bitField0_ |= 0x00000002;
- version_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional int32 version = 2;</code>
- */
- public Builder clearVersion() {
- bitField0_ = (bitField0_ & ~0x00000002);
- version_ = 0;
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:UserPayloadProto)
- }
-
- static {
- defaultInstance = new UserPayloadProto(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:UserPayloadProto)
- }
-
- public interface EntityDescriptorProtoOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional string class_name = 1;
- /**
- * <code>optional string class_name = 1;</code>
- */
- boolean hasClassName();
- /**
- * <code>optional string class_name = 1;</code>
- */
- java.lang.String getClassName();
- /**
- * <code>optional string class_name = 1;</code>
- */
- com.google.protobuf.ByteString
- getClassNameBytes();
-
- // optional .UserPayloadProto user_payload = 2;
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- boolean hasUserPayload();
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload();
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder();
-
- // optional bytes history_text = 3;
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- boolean hasHistoryText();
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- com.google.protobuf.ByteString getHistoryText();
- }
- /**
- * Protobuf type {@code EntityDescriptorProto}
- */
- public static final class EntityDescriptorProto extends
- com.google.protobuf.GeneratedMessage
- implements EntityDescriptorProtoOrBuilder {
- // Use EntityDescriptorProto.newBuilder() to construct.
- private EntityDescriptorProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private EntityDescriptorProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final EntityDescriptorProto defaultInstance;
- public static EntityDescriptorProto getDefaultInstance() {
- return defaultInstance;
- }
-
- public EntityDescriptorProto getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private EntityDescriptorProto(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- className_ = input.readBytes();
- break;
- }
- case 18: {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder subBuilder = null;
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- subBuilder = userPayload_.toBuilder();
- }
- userPayload_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.PARSER, extensionRegistry);
- if (subBuilder != null) {
- subBuilder.mergeFrom(userPayload_);
- userPayload_ = subBuilder.buildPartial();
- }
- bitField0_ |= 0x00000002;
- break;
- }
- case 26: {
- bitField0_ |= 0x00000004;
- historyText_ = input.readBytes();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
- }
-
- public static com.google.protobuf.Parser<EntityDescriptorProto> PARSER =
- new com.google.protobuf.AbstractParser<EntityDescriptorProto>() {
- public EntityDescriptorProto parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new EntityDescriptorProto(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<EntityDescriptorProto> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // optional string class_name = 1;
- public static final int CLASS_NAME_FIELD_NUMBER = 1;
- private java.lang.Object className_;
- /**
- * <code>optional string class_name = 1;</code>
- */
- public boolean hasClassName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public java.lang.String getClassName() {
- java.lang.Object ref = className_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- className_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public com.google.protobuf.ByteString
- getClassNameBytes() {
- java.lang.Object ref = className_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- className_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional .UserPayloadProto user_payload = 2;
- public static final int USER_PAYLOAD_FIELD_NUMBER = 2;
- private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_;
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public boolean hasUserPayload() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
- return userPayload_;
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
- return userPayload_;
- }
-
- // optional bytes history_text = 3;
- public static final int HISTORY_TEXT_FIELD_NUMBER = 3;
- private com.google.protobuf.ByteString historyText_;
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- public boolean hasHistoryText() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- public com.google.protobuf.ByteString getHistoryText() {
- return historyText_;
- }
-
- private void initFields() {
- className_ = "";
- userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
- historyText_ = com.google.protobuf.ByteString.EMPTY;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getClassNameBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeMessage(2, userPayload_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeBytes(3, historyText_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getClassNameBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(2, userPayload_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(3, historyText_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)) {
- return super.equals(obj);
- }
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) obj;
-
- boolean result = true;
- result = result && (hasClassName() == other.hasClassName());
- if (hasClassName()) {
- result = result && getClassName()
- .equals(other.getClassName());
- }
- result = result && (hasUserPayload() == other.hasUserPayload());
- if (hasUserPayload()) {
- result = result && getUserPayload()
- .equals(other.getUserPayload());
- }
- result = result && (hasHistoryText() == other.hasHistoryText());
- if (hasHistoryText()) {
- result = result && getHistoryText()
- .equals(other.getHistoryText());
- }
- result = result &&
- getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasClassName()) {
- hash = (37 * hash) + CLASS_NAME_FIELD_NUMBER;
- hash = (53 * hash) + getClassName().hashCode();
- }
- if (hasUserPayload()) {
- hash = (37 * hash) + USER_PAYLOAD_FIELD_NUMBER;
- hash = (53 * hash) + getUserPayload().hashCode();
- }
- if (hasHistoryText()) {
- hash = (37 * hash) + HISTORY_TEXT_FIELD_NUMBER;
- hash = (53 * hash) + getHistoryText().hashCode();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code EntityDescriptorProto}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- getUserPayloadFieldBuilder();
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- className_ = "";
- bitField0_ = (bitField0_ & ~0x00000001);
- if (userPayloadBuilder_ == null) {
- userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
- } else {
- userPayloadBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000002);
- historyText_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_EntityDescriptorProto_descriptor;
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getDefaultInstanceForType() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto build() {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto buildPartial() {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.className_ = className_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- if (userPayloadBuilder_ == null) {
- result.userPayload_ = userPayload_;
- } else {
- result.userPayload_ = userPayloadBuilder_.build();
- }
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.historyText_ = historyText_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) {
- return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto other) {
- if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) return this;
- if (other.hasClassName()) {
- bitField0_ |= 0x00000001;
- className_ = other.className_;
- onChanged();
- }
- if (other.hasUserPayload()) {
- mergeUserPayload(other.getUserPayload());
- }
- if (other.hasHistoryText()) {
- setHistoryText(other.getHistoryText());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // optional string class_name = 1;
- private java.lang.Object className_ = "";
- /**
- * <code>optional string class_name = 1;</code>
- */
- public boolean hasClassName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public java.lang.String getClassName() {
- java.lang.Object ref = className_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- className_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public com.google.protobuf.ByteString
- getClassNameBytes() {
- java.lang.Object ref = className_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- className_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public Builder setClassName(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- className_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public Builder clearClassName() {
- bitField0_ = (bitField0_ & ~0x00000001);
- className_ = getDefaultInstance().getClassName();
- onChanged();
- return this;
- }
- /**
- * <code>optional string class_name = 1;</code>
- */
- public Builder setClassNameBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- className_ = value;
- onChanged();
- return this;
- }
-
- // optional .UserPayloadProto user_payload = 2;
- private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder> userPayloadBuilder_;
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public boolean hasUserPayload() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto getUserPayload() {
- if (userPayloadBuilder_ == null) {
- return userPayload_;
- } else {
- return userPayloadBuilder_.getMessage();
- }
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public Builder setUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
- if (userPayloadBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- userPayload_ = value;
- onChanged();
- } else {
- userPayloadBuilder_.setMessage(value);
- }
- bitField0_ |= 0x00000002;
- return this;
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public Builder setUserPayload(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder builderForValue) {
- if (userPayloadBuilder_ == null) {
- userPayload_ = builderForValue.build();
- onChanged();
- } else {
- userPayloadBuilder_.setMessage(builderForValue.build());
- }
- bitField0_ |= 0x00000002;
- return this;
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public Builder mergeUserPayload(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto value) {
- if (userPayloadBuilder_ == null) {
- if (((bitField0_ & 0x00000002) == 0x00000002) &&
- userPayload_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance()) {
- userPayload_ =
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.newBuilder(userPayload_).mergeFrom(value).buildPartial();
- } else {
- userPayload_ = value;
- }
- onChanged();
- } else {
- userPayloadBuilder_.mergeFrom(value);
- }
- bitField0_ |= 0x00000002;
- return this;
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public Builder clearUserPayload() {
- if (userPayloadBuilder_ == null) {
- userPayload_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.getDefaultInstance();
- onChanged();
- } else {
- userPayloadBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000002);
- return this;
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder getUserPayloadBuilder() {
- bitField0_ |= 0x00000002;
- onChanged();
- return getUserPayloadFieldBuilder().getBuilder();
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder getUserPayloadOrBuilder() {
- if (userPayloadBuilder_ != null) {
- return userPayloadBuilder_.getMessageOrBuilder();
- } else {
- return userPayload_;
- }
- }
- /**
- * <code>optional .UserPayloadProto user_payload = 2;</code>
- */
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder>
- getUserPayloadFieldBuilder() {
- if (userPayloadBuilder_ == null) {
- userPayloadBuilder_ = new com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProtoOrBuilder>(
- userPayload_,
- getParentForChildren(),
- isClean());
- userPayload_ = null;
- }
- return userPayloadBuilder_;
- }
-
- // optional bytes history_text = 3;
- private com.google.protobuf.ByteString historyText_ = com.google.protobuf.ByteString.EMPTY;
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- public boolean hasHistoryText() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- public com.google.protobuf.ByteString getHistoryText() {
- return historyText_;
- }
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- public Builder setHistoryText(com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000004;
- historyText_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional bytes history_text = 3;</code>
- */
- public Builder clearHistoryText() {
- bitField0_ = (bitField0_ & ~0x00000004);
- historyText_ = getDefaultInstance().getHistoryText();
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:EntityDescriptorProto)
- }
-
- static {
- defaultInstance = new EntityDescriptorProto(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:EntityDescriptorProto)
- }
-
- public interface IOSpecProtoOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // optional string connected_vertex_name = 1;
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- boolean hasConnectedVertexName();
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- java.lang.String getConnectedVertexName();
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- com.google.protobuf.ByteString
- getConnectedVertexNameBytes();
-
- // optional .EntityDescriptorProto io_descriptor = 2;
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- boolean hasIoDescriptor();
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor();
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder();
-
- // optional int32 physical_edge_count = 3;
- /**
- * <code>optional int32 physical_edge_count = 3;</code>
- */
- boolean hasPhysicalEdgeCount();
- /**
- * <code>optional int32 physical_edge_count = 3;</code>
- */
- int getPhysicalEdgeCount();
- }
- /**
- * Protobuf type {@code IOSpecProto}
- */
- public static final class IOSpecProto extends
- com.google.protobuf.GeneratedMessage
- implements IOSpecProtoOrBuilder {
- // Use IOSpecProto.newBuilder() to construct.
- private IOSpecProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private IOSpecProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final IOSpecProto defaultInstance;
- public static IOSpecProto getDefaultInstance() {
- return defaultInstance;
- }
-
- public IOSpecProto getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private IOSpecProto(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- connectedVertexName_ = input.readBytes();
- break;
- }
- case 18: {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder subBuilder = null;
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- subBuilder = ioDescriptor_.toBuilder();
- }
- ioDescriptor_ = input.readMessage(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.PARSER, extensionRegistry);
- if (subBuilder != null) {
- subBuilder.mergeFrom(ioDescriptor_);
- ioDescriptor_ = subBuilder.buildPartial();
- }
- bitField0_ |= 0x00000002;
- break;
- }
- case 24: {
- bitField0_ |= 0x00000004;
- physicalEdgeCount_ = input.readInt32();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
- }
-
- public static com.google.protobuf.Parser<IOSpecProto> PARSER =
- new com.google.protobuf.AbstractParser<IOSpecProto>() {
- public IOSpecProto parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new IOSpecProto(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<IOSpecProto> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // optional string connected_vertex_name = 1;
- public static final int CONNECTED_VERTEX_NAME_FIELD_NUMBER = 1;
- private java.lang.Object connectedVertexName_;
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public boolean hasConnectedVertexName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public java.lang.String getConnectedVertexName() {
- java.lang.Object ref = connectedVertexName_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- connectedVertexName_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public com.google.protobuf.ByteString
- getConnectedVertexNameBytes() {
- java.lang.Object ref = connectedVertexName_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- connectedVertexName_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional .EntityDescriptorProto io_descriptor = 2;
- public static final int IO_DESCRIPTOR_FIELD_NUMBER = 2;
- private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_;
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public boolean hasIoDescriptor() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
- return ioDescriptor_;
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
- return ioDescriptor_;
- }
-
- // optional int32 physical_edge_count = 3;
- public static final int PHYSICAL_EDGE_COUNT_FIELD_NUMBER = 3;
- private int physicalEdgeCount_;
- /**
- * <code>optional int32 physical_edge_count = 3;</code>
- */
- public boolean hasPhysicalEdgeCount() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>optional int32 physical_edge_count = 3;</code>
- */
- public int getPhysicalEdgeCount() {
- return physicalEdgeCount_;
- }
-
- private void initFields() {
- connectedVertexName_ = "";
- ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
- physicalEdgeCount_ = 0;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getConnectedVertexNameBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeMessage(2, ioDescriptor_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeInt32(3, physicalEdgeCount_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getConnectedVertexNameBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(2, ioDescriptor_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(3, physicalEdgeCount_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)) {
- return super.equals(obj);
- }
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) obj;
-
- boolean result = true;
- result = result && (hasConnectedVertexName() == other.hasConnectedVertexName());
- if (hasConnectedVertexName()) {
- result = result && getConnectedVertexName()
- .equals(other.getConnectedVertexName());
- }
- result = result && (hasIoDescriptor() == other.hasIoDescriptor());
- if (hasIoDescriptor()) {
- result = result && getIoDescriptor()
- .equals(other.getIoDescriptor());
- }
- result = result && (hasPhysicalEdgeCount() == other.hasPhysicalEdgeCount());
- if (hasPhysicalEdgeCount()) {
- result = result && (getPhysicalEdgeCount()
- == other.getPhysicalEdgeCount());
- }
- result = result &&
- getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasConnectedVertexName()) {
- hash = (37 * hash) + CONNECTED_VERTEX_NAME_FIELD_NUMBER;
- hash = (53 * hash) + getConnectedVertexName().hashCode();
- }
- if (hasIoDescriptor()) {
- hash = (37 * hash) + IO_DESCRIPTOR_FIELD_NUMBER;
- hash = (53 * hash) + getIoDescriptor().hashCode();
- }
- if (hasPhysicalEdgeCount()) {
- hash = (37 * hash) + PHYSICAL_EDGE_COUNT_FIELD_NUMBER;
- hash = (53 * hash) + getPhysicalEdgeCount();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code IOSpecProto}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProtoOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.class, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- getIoDescriptorFieldBuilder();
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- connectedVertexName_ = "";
- bitField0_ = (bitField0_ & ~0x00000001);
- if (ioDescriptorBuilder_ == null) {
- ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
- } else {
- ioDescriptorBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000002);
- physicalEdgeCount_ = 0;
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.internal_static_IOSpecProto_descriptor;
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto getDefaultInstanceForType() {
- return org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto build() {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto buildPartial() {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto result = new org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.connectedVertexName_ = connectedVertexName_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- if (ioDescriptorBuilder_ == null) {
- result.ioDescriptor_ = ioDescriptor_;
- } else {
- result.ioDescriptor_ = ioDescriptorBuilder_.build();
- }
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.physicalEdgeCount_ = physicalEdgeCount_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) {
- return mergeFrom((org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto other) {
- if (other == org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto.getDefaultInstance()) return this;
- if (other.hasConnectedVertexName()) {
- bitField0_ |= 0x00000001;
- connectedVertexName_ = other.connectedVertexName_;
- onChanged();
- }
- if (other.hasIoDescriptor()) {
- mergeIoDescriptor(other.getIoDescriptor());
- }
- if (other.hasPhysicalEdgeCount()) {
- setPhysicalEdgeCount(other.getPhysicalEdgeCount());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // optional string connected_vertex_name = 1;
- private java.lang.Object connectedVertexName_ = "";
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public boolean hasConnectedVertexName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public java.lang.String getConnectedVertexName() {
- java.lang.Object ref = connectedVertexName_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- connectedVertexName_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public com.google.protobuf.ByteString
- getConnectedVertexNameBytes() {
- java.lang.Object ref = connectedVertexName_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- connectedVertexName_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public Builder setConnectedVertexName(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- connectedVertexName_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public Builder clearConnectedVertexName() {
- bitField0_ = (bitField0_ & ~0x00000001);
- connectedVertexName_ = getDefaultInstance().getConnectedVertexName();
- onChanged();
- return this;
- }
- /**
- * <code>optional string connected_vertex_name = 1;</code>
- */
- public Builder setConnectedVertexNameBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- connectedVertexName_ = value;
- onChanged();
- return this;
- }
-
- // optional .EntityDescriptorProto io_descriptor = 2;
- private org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder> ioDescriptorBuilder_;
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public boolean hasIoDescriptor() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto getIoDescriptor() {
- if (ioDescriptorBuilder_ == null) {
- return ioDescriptor_;
- } else {
- return ioDescriptorBuilder_.getMessage();
- }
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public Builder setIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
- if (ioDescriptorBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ioDescriptor_ = value;
- onChanged();
- } else {
- ioDescriptorBuilder_.setMessage(value);
- }
- bitField0_ |= 0x00000002;
- return this;
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public Builder setIoDescriptor(
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder builderForValue) {
- if (ioDescriptorBuilder_ == null) {
- ioDescriptor_ = builderForValue.build();
- onChanged();
- } else {
- ioDescriptorBuilder_.setMessage(builderForValue.build());
- }
- bitField0_ |= 0x00000002;
- return this;
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public Builder mergeIoDescriptor(org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto value) {
- if (ioDescriptorBuilder_ == null) {
- if (((bitField0_ & 0x00000002) == 0x00000002) &&
- ioDescriptor_ != org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance()) {
- ioDescriptor_ =
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.newBuilder(ioDescriptor_).mergeFrom(value).buildPartial();
- } else {
- ioDescriptor_ = value;
- }
- onChanged();
- } else {
- ioDescriptorBuilder_.mergeFrom(value);
- }
- bitField0_ |= 0x00000002;
- return this;
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public Builder clearIoDescriptor() {
- if (ioDescriptorBuilder_ == null) {
- ioDescriptor_ = org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.getDefaultInstance();
- onChanged();
- } else {
- ioDescriptorBuilder_.clear();
- }
- bitField0_ = (bitField0_ & ~0x00000002);
- return this;
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder getIoDescriptorBuilder() {
- bitField0_ |= 0x00000002;
- onChanged();
- return getIoDescriptorFieldBuilder().getBuilder();
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- public org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder getIoDescriptorOrBuilder() {
- if (ioDescriptorBuilder_ != null) {
- return ioDescriptorBuilder_.getMessageOrBuilder();
- } else {
- return ioDescriptor_;
- }
- }
- /**
- * <code>optional .EntityDescriptorProto io_descriptor = 2;</code>
- */
- private com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder>
- getIoDescriptorFieldBuilder() {
- if (ioDescriptorBuilder_ == null) {
- ioDescriptorBuilder_ = new com.google.protobuf.SingleFieldBuilder<
- org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto.Builder, org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProtoOrBuilder>(
- ioDescriptor_,
- getParentForChildren(),
- isClean());
- ioDescriptor_ = null;
- }
- return ioDescriptorBuilder_;
- }
-
- // optional int32 physical_edge_count = 3;
- private int physicalEdgeCount_ ;
- /**
- * <code>optional int32 physical_edge
<TRUNCATED>
[06/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
deleted file mode 100644
index 515bf3c..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapNodeId.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-public class LlapNodeId {
-
- private static final LoadingCache<LlapNodeId, LlapNodeId> CACHE =
- CacheBuilder.newBuilder().softValues().build(
- new CacheLoader<LlapNodeId, LlapNodeId>() {
- @Override
- public LlapNodeId load(LlapNodeId key) throws Exception {
- return key;
- }
- });
-
- public static LlapNodeId getInstance(String hostname, int port) {
- return CACHE.getUnchecked(new LlapNodeId(hostname, port));
- }
-
-
- private final String hostname;
- private final int port;
-
-
- private LlapNodeId(String hostname, int port) {
- this.hostname = hostname;
- this.port = port;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public int getPort() {
- return port;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- LlapNodeId that = (LlapNodeId) o;
-
- if (port != that.port) {
- return false;
- }
- if (!hostname.equals(that.hostname)) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = hostname.hashCode();
- result = 1009 * result + port;
- return result;
- }
-
- @Override
- public String toString() {
- return hostname + ":" + port;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 544af09..0399798 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -25,6 +25,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Collection;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -217,7 +219,13 @@ public class LlapServiceDriver {
CompressionUtils.unTar(new Path(libDir, "tez.tar.gz").toString(), libDir.toString(), true);
lfs.delete(new Path(libDir, "tez.tar.gz"), false);
+ // llap-common
+ lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapDaemonProtocolProtos.class)), libDir);
+ // llap-tez
+ lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapTezUtils.class)), libDir);
+ // llap-server
lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(LlapInputFormat.class)), libDir);
+ // hive-exec
lfs.copyFromLocalFile(new Path(Utilities.jarFinderGetJar(HiveInputFormat.class)), libDir);
// copy default aux classes (json/hbase)
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
deleted file mode 100644
index 4c09941..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapDaemonProtocolBlockingPB.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon;
-
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.hive.llap.security.LlapTokenSelector;
-
-@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB", protocolVersion = 1)
-@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
-@TokenInfo(LlapTokenSelector.class)
-public interface LlapDaemonProtocolBlockingPB extends LlapDaemonProtocolProtos.LlapDaemonProtocol.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java
deleted file mode 100644
index 4efadac..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/LlapManagementProtocolBlockingPB.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.llap.daemon;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-
-@ProtocolInfo(protocolName = "org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB", protocolVersion = 1)
-@KerberosInfo(serverPrincipal = HiveConf.HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME)
-public interface LlapManagementProtocolBlockingPB extends LlapDaemonProtocolProtos.LlapManagementProtocol.BlockingInterface {
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 98951e1..37120d2 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -65,7 +65,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
private static final Logger LOG = LoggerFactory.getLogger(LlapDaemon.class);
private final Configuration shuffleHandlerConf;
- private final LlapDaemonProtocolServerImpl server;
+ private final LlapProtocolServerImpl server;
private final ContainerRunnerImpl containerRunner;
private final AMReporter amReporter;
private final LlapRegistryService registry;
@@ -164,7 +164,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
this.amReporter = new AMReporter(srvAddress, new QueryFailedHandlerProxy(), daemonConf);
- this.server = new LlapDaemonProtocolServerImpl(
+ this.server = new LlapProtocolServerImpl(
numHandlers, this, srvAddress, mngAddress, srvPort, mngPort);
this.containerRunner = new ContainerRunnerImpl(daemonConf,
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
deleted file mode 100644
index 9c7d2e2..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolClientImpl.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon.impl;
-
-import javax.annotation.Nullable;
-import javax.net.SocketFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.ProtocolProxy;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.security.UserGroupInformation;
-
-// TODO Change all this to be based on a regular interface instead of relying on the Proto service - Exception signatures cannot be controlled without this for the moment.
-
-
-public class LlapDaemonProtocolClientImpl implements LlapDaemonProtocolBlockingPB {
-
- private final Configuration conf;
- private final InetSocketAddress serverAddr;
- private final RetryPolicy retryPolicy;
- private final SocketFactory socketFactory;
- LlapDaemonProtocolBlockingPB proxy;
-
-
- public LlapDaemonProtocolClientImpl(Configuration conf, String hostname, int port,
- @Nullable RetryPolicy retryPolicy,
- @Nullable SocketFactory socketFactory) {
- this.conf = conf;
- this.serverAddr = NetUtils.createSocketAddr(hostname, port);
- this.retryPolicy = retryPolicy;
- if (socketFactory == null) {
- this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
- } else {
- this.socketFactory = socketFactory;
- }
- }
-
- @Override
- public SubmitWorkResponseProto submitWork(RpcController controller,
- SubmitWorkRequestProto request) throws
- ServiceException {
- try {
- return getProxy().submitWork(null, request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
- SourceStateUpdatedRequestProto request) throws
- ServiceException {
- try {
- return getProxy().sourceStateUpdated(null, request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public QueryCompleteResponseProto queryComplete(RpcController controller,
- QueryCompleteRequestProto request) throws
- ServiceException {
- try {
- return getProxy().queryComplete(null, request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public TerminateFragmentResponseProto terminateFragment(
- RpcController controller,
- TerminateFragmentRequestProto request) throws ServiceException {
- try {
- return getProxy().terminateFragment(null, request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- public LlapDaemonProtocolBlockingPB getProxy() throws IOException {
- if (proxy == null) {
- proxy = createProxy();
- }
- return proxy;
- }
-
- public LlapDaemonProtocolBlockingPB createProxy() throws IOException {
- RPC.setProtocolEngine(conf, LlapDaemonProtocolBlockingPB.class, ProtobufRpcEngine.class);
- ProtocolProxy<LlapDaemonProtocolBlockingPB> proxy =
- RPC.getProtocolProxy(LlapDaemonProtocolBlockingPB.class, 0, serverAddr,
- UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
- retryPolicy);
- return proxy.getProxy();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
deleted file mode 100644
index 45ca906..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon.impl;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedAction;
-import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.ByteArrayDataOutput;
-import com.google.common.io.ByteStreams;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.hive.llap.security.LlapSecurityHelper;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.hive.llap.security.SecretManager;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.security.LlapDaemonPolicyProvider;
-
-public class LlapDaemonProtocolServerImpl extends AbstractService
- implements LlapDaemonProtocolBlockingPB, LlapManagementProtocolBlockingPB {
-
- private static final Logger LOG = LoggerFactory.getLogger(LlapDaemonProtocolServerImpl.class);
-
- private final int numHandlers;
- private final ContainerRunner containerRunner;
- private final int srvPort, mngPort;
- private RPC.Server server, mngServer;
- private final AtomicReference<InetSocketAddress> srvAddress, mngAddress;
- private SecretManager zkSecretManager;
-
- public LlapDaemonProtocolServerImpl(int numHandlers,
- ContainerRunner containerRunner,
- AtomicReference<InetSocketAddress> srvAddress,
- AtomicReference<InetSocketAddress> mngAddress,
- int srvPort,
- int mngPort) {
- super("LlapDaemonProtocolServerImpl");
- this.numHandlers = numHandlers;
- this.containerRunner = containerRunner;
- this.srvAddress = srvAddress;
- this.srvPort = srvPort;
- this.mngAddress = mngAddress;
- this.mngPort = mngPort;
- LOG.info("Creating: " + LlapDaemonProtocolServerImpl.class.getSimpleName() +
- " with port configured to: " + srvPort);
- }
-
- @Override
- public SubmitWorkResponseProto submitWork(RpcController controller,
- SubmitWorkRequestProto request) throws
- ServiceException {
- try {
- return containerRunner.submitWork(request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
- SourceStateUpdatedRequestProto request) throws ServiceException {
- return containerRunner.sourceStateUpdated(request);
- }
-
- @Override
- public QueryCompleteResponseProto queryComplete(RpcController controller,
- QueryCompleteRequestProto request) throws ServiceException {
- return containerRunner.queryComplete(request);
- }
-
- @Override
- public TerminateFragmentResponseProto terminateFragment(
- RpcController controller,
- TerminateFragmentRequestProto request) throws ServiceException {
- return containerRunner.terminateFragment(request);
- }
-
- @Override
- public void serviceStart() {
- final Configuration conf = getConfig();
- final BlockingService daemonImpl =
- LlapDaemonProtocolProtos.LlapDaemonProtocol.newReflectiveBlockingService(this);
- final BlockingService managementImpl =
- LlapDaemonProtocolProtos.LlapManagementProtocol.newReflectiveBlockingService(this);
- if (!UserGroupInformation.isSecurityEnabled()) {
- startProtocolServers(conf, daemonImpl, managementImpl);
- return;
- }
- String llapPrincipal = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_PRINCIPAL),
- llapKeytab = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_KEYTAB_FILE);
- zkSecretManager = SecretManager.createSecretManager(conf, llapPrincipal, llapKeytab);
-
- // Start the protocol server after properly authenticating with daemon keytab.
- UserGroupInformation daemonUgi = null;
- try {
- daemonUgi = LlapSecurityHelper.loginWithKerberos(llapPrincipal, llapKeytab);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- daemonUgi.doAs(new PrivilegedAction<Void>() {
- @Override
- public Void run() {
- startProtocolServers(conf, daemonImpl, managementImpl);
- return null;
- }
- });
- }
-
- private void startProtocolServers(
- Configuration conf, BlockingService daemonImpl, BlockingService managementImpl) {
- server = startProtocolServer(srvPort, numHandlers, srvAddress, conf, daemonImpl,
- LlapDaemonProtocolBlockingPB.class);
- mngServer = startProtocolServer(mngPort, 2, mngAddress, conf, managementImpl,
- LlapManagementProtocolBlockingPB.class);
- }
-
- private RPC.Server startProtocolServer(int srvPort, int numHandlers,
- AtomicReference<InetSocketAddress> bindAddress, Configuration conf,
- BlockingService impl, Class<?> protocolClass) {
- InetSocketAddress addr = new InetSocketAddress(srvPort);
- RPC.Server server;
- try {
- server = createServer(protocolClass, addr, conf, numHandlers, impl);
- server.start();
- } catch (IOException e) {
- LOG.error("Failed to run RPC Server on port: " + srvPort, e);
- throw new RuntimeException(e);
- }
-
- InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server);
- bindAddress.set(NetUtils.createSocketAddrForHost(
- serverBindAddress.getAddress().getCanonicalHostName(),
- serverBindAddress.getPort()));
- LOG.info("Instantiated " + protocolClass.getSimpleName() + " at " + bindAddress);
- return server;
- }
-
- @Override
- public void serviceStop() {
- if (server != null) {
- server.stop();
- }
- if (mngServer != null) {
- mngServer.stop();
- }
- }
-
- @InterfaceAudience.Private
- @VisibleForTesting
- InetSocketAddress getBindAddress() {
- return srvAddress.get();
- }
-
- private RPC.Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
- int numHandlers, BlockingService blockingService) throws
- IOException {
- RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
- RPC.Builder builder = new RPC.Builder(conf)
- .setProtocol(pbProtocol)
- .setInstance(blockingService)
- .setBindAddress(addr.getHostName())
- .setPort(addr.getPort())
- .setNumHandlers(numHandlers);
- if (zkSecretManager != null) {
- builder = builder.setSecretManager(zkSecretManager);
- }
- RPC.Server server = builder.build();
-
- if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
- server.refreshServiceAcl(conf, new LlapDaemonPolicyProvider());
- }
- return server;
- }
-
-
- @Override
- public GetTokenResponseProto getDelegationToken(RpcController controller,
- GetTokenRequestProto request) throws ServiceException {
- if (zkSecretManager == null) {
- throw new ServiceException("Operation not supported on unsecure cluster");
- }
- UserGroupInformation ugi;
- try {
- ugi = UserGroupInformation.getCurrentUser();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- String user = ugi.getUserName();
- Text owner = new Text(user);
- Text realUser = null;
- if (ugi.getRealUser() != null) {
- realUser = new Text(ugi.getRealUser().getUserName());
- }
- Text renewer = new Text(ugi.getShortUserName());
- LlapTokenIdentifier llapId = new LlapTokenIdentifier(owner, renewer, realUser);
- // TODO: note that the token is not renewable right now and will last for 2 weeks by default.
- Token<LlapTokenIdentifier> token = new Token<LlapTokenIdentifier>(llapId, zkSecretManager);
- ByteArrayDataOutput out = ByteStreams.newDataOutput();
- try {
- token.write(out);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- ByteString bs = ByteString.copyFrom(out.toByteArray());
- GetTokenResponseProto response = GetTokenResponseProto.newBuilder().setToken(bs).build();
- return response;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java
deleted file mode 100644
index e293a95..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapManagementProtocolClientImpl.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.daemon.impl;
-
-import javax.annotation.Nullable;
-import javax.net.SocketFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.ProtocolProxy;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
-import org.apache.hadoop.security.UserGroupInformation;
-
-public class LlapManagementProtocolClientImpl implements LlapManagementProtocolBlockingPB {
-
- private final Configuration conf;
- private final InetSocketAddress serverAddr;
- private final RetryPolicy retryPolicy;
- private final SocketFactory socketFactory;
- LlapManagementProtocolBlockingPB proxy;
-
-
- public LlapManagementProtocolClientImpl(Configuration conf, String hostname, int port,
- @Nullable RetryPolicy retryPolicy,
- @Nullable SocketFactory socketFactory) {
- this.conf = conf;
- this.serverAddr = NetUtils.createSocketAddr(hostname, port);
- this.retryPolicy = retryPolicy;
- if (socketFactory == null) {
- this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
- } else {
- this.socketFactory = socketFactory;
- }
- }
-
- public LlapManagementProtocolBlockingPB getProxy() throws IOException {
- if (proxy == null) {
- proxy = createProxy();
- }
- return proxy;
- }
-
- public LlapManagementProtocolBlockingPB createProxy() throws IOException {
- RPC.setProtocolEngine(conf, LlapManagementProtocolBlockingPB.class, ProtobufRpcEngine.class);
- ProtocolProxy<LlapManagementProtocolBlockingPB> proxy =
- RPC.getProtocolProxy(LlapManagementProtocolBlockingPB.class, 0, serverAddr,
- UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), 0,
- retryPolicy);
- return proxy.getProxy();
- }
-
- @Override
- public GetTokenResponseProto getDelegationToken(RpcController controller,
- GetTokenRequestProto request) throws ServiceException {
- try {
- return getProxy().getDelegationToken(null, request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
new file mode 100644
index 0000000..c386d77
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap.daemon.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteArrayDataOutput;
+import com.google.common.io.ByteStreams;
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.hive.llap.security.LlapSecurityHelper;
+import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
+import org.apache.hadoop.hive.llap.security.SecretManager;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
+import org.apache.hadoop.hive.llap.security.LlapDaemonPolicyProvider;
+
+public class LlapProtocolServerImpl extends AbstractService
+ implements LlapProtocolBlockingPB, LlapManagementProtocolPB {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LlapProtocolServerImpl.class);
+
+ private final int numHandlers;
+ private final ContainerRunner containerRunner;
+ private final int srvPort, mngPort;
+ private RPC.Server server, mngServer;
+ private final AtomicReference<InetSocketAddress> srvAddress, mngAddress;
+ private SecretManager zkSecretManager;
+
+ public LlapProtocolServerImpl(int numHandlers,
+ ContainerRunner containerRunner,
+ AtomicReference<InetSocketAddress> srvAddress,
+ AtomicReference<InetSocketAddress> mngAddress,
+ int srvPort,
+ int mngPort) {
+ super("LlapDaemonProtocolServerImpl");
+ this.numHandlers = numHandlers;
+ this.containerRunner = containerRunner;
+ this.srvAddress = srvAddress;
+ this.srvPort = srvPort;
+ this.mngAddress = mngAddress;
+ this.mngPort = mngPort;
+ LOG.info("Creating: " + LlapProtocolServerImpl.class.getSimpleName() +
+ " with port configured to: " + srvPort);
+ }
+
+ @Override
+ public SubmitWorkResponseProto submitWork(RpcController controller,
+ SubmitWorkRequestProto request) throws
+ ServiceException {
+ try {
+ return containerRunner.submitWork(request);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public SourceStateUpdatedResponseProto sourceStateUpdated(RpcController controller,
+ SourceStateUpdatedRequestProto request) throws ServiceException {
+ return containerRunner.sourceStateUpdated(request);
+ }
+
+ @Override
+ public QueryCompleteResponseProto queryComplete(RpcController controller,
+ QueryCompleteRequestProto request) throws ServiceException {
+ return containerRunner.queryComplete(request);
+ }
+
+ @Override
+ public TerminateFragmentResponseProto terminateFragment(
+ RpcController controller,
+ TerminateFragmentRequestProto request) throws ServiceException {
+ return containerRunner.terminateFragment(request);
+ }
+
+ @Override
+ public void serviceStart() {
+ final Configuration conf = getConfig();
+ final BlockingService daemonImpl =
+ LlapDaemonProtocolProtos.LlapDaemonProtocol.newReflectiveBlockingService(this);
+ final BlockingService managementImpl =
+ LlapDaemonProtocolProtos.LlapManagementProtocol.newReflectiveBlockingService(this);
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ startProtocolServers(conf, daemonImpl, managementImpl);
+ return;
+ }
+ String llapPrincipal = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_PRINCIPAL),
+ llapKeytab = HiveConf.getVar(conf, ConfVars.LLAP_KERBEROS_KEYTAB_FILE);
+ zkSecretManager = SecretManager.createSecretManager(conf, llapPrincipal, llapKeytab);
+
+ // Start the protocol server after properly authenticating with daemon keytab.
+ UserGroupInformation daemonUgi = null;
+ try {
+ daemonUgi = LlapSecurityHelper.loginWithKerberos(llapPrincipal, llapKeytab);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ daemonUgi.doAs(new PrivilegedAction<Void>() {
+ @Override
+ public Void run() {
+ startProtocolServers(conf, daemonImpl, managementImpl);
+ return null;
+ }
+ });
+ }
+
+ private void startProtocolServers(
+ Configuration conf, BlockingService daemonImpl, BlockingService managementImpl) {
+ server = startProtocolServer(srvPort, numHandlers, srvAddress, conf, daemonImpl,
+ LlapProtocolBlockingPB.class);
+ mngServer = startProtocolServer(mngPort, 2, mngAddress, conf, managementImpl,
+ LlapManagementProtocolPB.class);
+ }
+
+ private RPC.Server startProtocolServer(int srvPort, int numHandlers,
+ AtomicReference<InetSocketAddress> bindAddress, Configuration conf,
+ BlockingService impl, Class<?> protocolClass) {
+ InetSocketAddress addr = new InetSocketAddress(srvPort);
+ RPC.Server server;
+ try {
+ server = createServer(protocolClass, addr, conf, numHandlers, impl);
+ server.start();
+ } catch (IOException e) {
+ LOG.error("Failed to run RPC Server on port: " + srvPort, e);
+ throw new RuntimeException(e);
+ }
+
+ InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server);
+ bindAddress.set(NetUtils.createSocketAddrForHost(
+ serverBindAddress.getAddress().getCanonicalHostName(),
+ serverBindAddress.getPort()));
+ LOG.info("Instantiated " + protocolClass.getSimpleName() + " at " + bindAddress);
+ return server;
+ }
+
+ @Override
+ public void serviceStop() {
+ if (server != null) {
+ server.stop();
+ }
+ if (mngServer != null) {
+ mngServer.stop();
+ }
+ }
+
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ InetSocketAddress getBindAddress() {
+ return srvAddress.get();
+ }
+
+ private RPC.Server createServer(Class<?> pbProtocol, InetSocketAddress addr, Configuration conf,
+ int numHandlers, BlockingService blockingService) throws
+ IOException {
+ RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class);
+ RPC.Builder builder = new RPC.Builder(conf)
+ .setProtocol(pbProtocol)
+ .setInstance(blockingService)
+ .setBindAddress(addr.getHostName())
+ .setPort(addr.getPort())
+ .setNumHandlers(numHandlers);
+ if (zkSecretManager != null) {
+ builder = builder.setSecretManager(zkSecretManager);
+ }
+ RPC.Server server = builder.build();
+
+ if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
+ server.refreshServiceAcl(conf, new LlapDaemonPolicyProvider());
+ }
+ return server;
+ }
+
+
+ @Override
+ public GetTokenResponseProto getDelegationToken(RpcController controller,
+ GetTokenRequestProto request) throws ServiceException {
+ if (zkSecretManager == null) {
+ throw new ServiceException("Operation not supported on unsecure cluster");
+ }
+ UserGroupInformation ugi;
+ try {
+ ugi = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ String user = ugi.getUserName();
+ Text owner = new Text(user);
+ Text realUser = null;
+ if (ugi.getRealUser() != null) {
+ realUser = new Text(ugi.getRealUser().getUserName());
+ }
+ Text renewer = new Text(ugi.getShortUserName());
+ LlapTokenIdentifier llapId = new LlapTokenIdentifier(owner, renewer, realUser);
+ // TODO: note that the token is not renewable right now and will last for 2 weeks by default.
+ Token<LlapTokenIdentifier> token = new Token<LlapTokenIdentifier>(llapId, zkSecretManager);
+ ByteArrayDataOutput out = ByteStreams.newDataOutput();
+ try {
+ token.write(out);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ ByteString bs = ByteString.copyFrom(out.toByteArray());
+ GetTokenResponseProto response = GetTokenResponseProto.newBuilder().setToken(bs).build();
+ return response;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
index aa065a9..480a394 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/QueryFragmentInfo.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.llap.daemon.FinishableStateUpdateHandler;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
+import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,7 +89,7 @@ public class QueryFragmentInfo {
boolean canFinish = true;
if (inputSpecList != null && !inputSpecList.isEmpty()) {
for (IOSpecProto inputSpec : inputSpecList) {
- if (isSourceOfInterest(inputSpec)) {
+ if (LlapTezUtils.isSourceOfInterest(inputSpec.getIoDescriptor().getClassName())) {
// Lookup the state in the map.
LlapDaemonProtocolProtos.SourceStateProto state = queryInfo.getSourceStateMap()
.get(inputSpec.getConnectedVertexName());
@@ -129,7 +129,7 @@ public class QueryFragmentInfo {
List<IOSpecProto> inputSpecList = fragmentSpec.getInputSpecsList();
if (inputSpecList != null && !inputSpecList.isEmpty()) {
for (IOSpecProto inputSpec : inputSpecList) {
- if (isSourceOfInterest(inputSpec)) {
+ if (LlapTezUtils.isSourceOfInterest(inputSpec.getIoDescriptor().getClassName())) {
sourcesOfInterest.add(inputSpec.getConnectedVertexName());
}
}
@@ -143,13 +143,6 @@ public class QueryFragmentInfo {
queryInfo.unregisterFinishableStateUpdate(handler);
}
- private boolean isSourceOfInterest(IOSpecProto inputSpec) {
- String inputClassName = inputSpec.getIoDescriptor().getClassName();
- // MRInput is not of interest since it'll always be ready.
- return !inputClassName.equals(MRInputLegacy.class.getName());
- }
-
-
@Override
public boolean equals(Object o) {
if (this == o) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
index ede2a03..d88d82a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecPro
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
-import org.apache.hadoop.hive.llap.tezplugins.Converters;
+import org.apache.hadoop.hive.llap.tez.Converters;
import org.apache.hadoop.hive.ql.io.IOContextMap;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java b/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
deleted file mode 100644
index 9549567..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/protocol/LlapTaskUmbilicalProtocol.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.security.token.TokenInfo;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
-import org.apache.tez.runtime.common.security.JobTokenSelector;
-
-@TokenInfo(JobTokenSelector.class)
-public interface LlapTaskUmbilicalProtocol extends VersionedProtocol {
-
- public static final long versionID = 1L;
-
- // From Tez. Eventually changes over to the LLAP protocol and ProtocolBuffers
- boolean canCommit(TezTaskAttemptID taskid) throws IOException;
- public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request)
- throws IOException, TezException;
-
- public void nodeHeartbeat(Text hostname, int port) throws IOException;
-
- public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
index d67647b..bedd265 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapDaemonPolicyProvider.java
@@ -18,17 +18,17 @@
package org.apache.hadoop.hive.llap.security;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
public class LlapDaemonPolicyProvider extends PolicyProvider {
private static final Service[] services = new Service[] {
new Service(HiveConf.ConfVars.LLAP_SECURITY_ACL.varname,
- LlapDaemonProtocolBlockingPB.class),
+ LlapProtocolBlockingPB.class),
new Service(HiveConf.ConfVars.LLAP_MANAGEMENT_ACL.varname,
- LlapManagementProtocolBlockingPB.class)
+ LlapManagementProtocolPB.class)
};
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
index a00b631..aa8745d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
import javax.net.SocketFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.llap.daemon.impl.LlapManagementProtocolClientImpl;
+import org.apache.hadoop.hive.llap.impl.LlapManagementProtocolClientImpl;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GetTokenRequestProto;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.hive.llap.registry.ServiceInstance;
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
index 4dca2ce..eb514f2 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapServerSecurityInfo.java
@@ -20,8 +20,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.LlapManagementProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapManagementProtocolPB;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -36,8 +36,8 @@ public class LlapServerSecurityInfo extends SecurityInfo {
if (LOG.isDebugEnabled()) {
LOG.debug("Trying to get KerberosInfo for " + protocol);
}
- if (!LlapDaemonProtocolBlockingPB.class.isAssignableFrom(protocol)
- && !LlapManagementProtocolBlockingPB.class.isAssignableFrom(protocol)) return null;
+ if (!LlapProtocolBlockingPB.class.isAssignableFrom(protocol)
+ && !LlapManagementProtocolPB.class.isAssignableFrom(protocol)) return null;
return new KerberosInfo() {
@Override
public Class<? extends Annotation> annotationType() {
@@ -62,7 +62,7 @@ public class LlapServerSecurityInfo extends SecurityInfo {
LOG.debug("Trying to get TokenInfo for " + protocol);
}
// Tokens cannot be used for the management protocol (for now).
- if (!LlapDaemonProtocolBlockingPB.class.isAssignableFrom(protocol)) return null;
+ if (!LlapProtocolBlockingPB.class.isAssignableFrom(protocol)) return null;
return new TokenInfo() {
@Override
public Class<? extends Annotation> annotationType() {
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
deleted file mode 100644
index b6e7499..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapTokenSelector.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import java.util.Collection;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
-
-public class LlapTokenSelector implements TokenSelector<LlapTokenIdentifier> {
- private static final Log LOG = LogFactory.getLog(LlapTokenSelector.class);
-
- @Override
- public Token<LlapTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
- if (service == null) return null;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Looking for a token with service " + service);
- }
- for (Token<? extends TokenIdentifier> token : tokens) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Token = " + token.getKind() + "; service = " + token.getService());
- }
- if (LlapTokenIdentifier.KIND_NAME.equals(token.getKind())
- && service.equals(token.getService())) {
- @SuppressWarnings("unchecked")
- Token<LlapTokenIdentifier> result = (Token<LlapTokenIdentifier>)token;
- return result;
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java
deleted file mode 100644
index f61d62f..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/Converters.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.EntityDescriptorProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.GroupInputSpecProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.IOSpecProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentSpecProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.UserPayloadProto;
-import org.apache.tez.common.TezCommonUtils;
-import org.apache.tez.dag.api.EntityDescriptor;
-import org.apache.tez.dag.api.InputDescriptor;
-import org.apache.tez.dag.api.OutputDescriptor;
-import org.apache.tez.dag.api.ProcessorDescriptor;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.dag.api.event.VertexState;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.GroupInputSpec;
-import org.apache.tez.runtime.api.impl.InputSpec;
-import org.apache.tez.runtime.api.impl.OutputSpec;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-
-public class Converters {
-
- public static TaskSpec getTaskSpecfromProto(FragmentSpecProto FragmentSpecProto) {
- TezTaskAttemptID taskAttemptID =
- TezTaskAttemptID.fromString(FragmentSpecProto.getFragmentIdentifierString());
-
- ProcessorDescriptor processorDescriptor = null;
- if (FragmentSpecProto.hasProcessorDescriptor()) {
- processorDescriptor = convertProcessorDescriptorFromProto(
- FragmentSpecProto.getProcessorDescriptor());
- }
-
- List<InputSpec> inputSpecList = new ArrayList<InputSpec>(FragmentSpecProto.getInputSpecsCount());
- if (FragmentSpecProto.getInputSpecsCount() > 0) {
- for (IOSpecProto inputSpecProto : FragmentSpecProto.getInputSpecsList()) {
- inputSpecList.add(getInputSpecFromProto(inputSpecProto));
- }
- }
-
- List<OutputSpec> outputSpecList =
- new ArrayList<OutputSpec>(FragmentSpecProto.getOutputSpecsCount());
- if (FragmentSpecProto.getOutputSpecsCount() > 0) {
- for (IOSpecProto outputSpecProto : FragmentSpecProto.getOutputSpecsList()) {
- outputSpecList.add(getOutputSpecFromProto(outputSpecProto));
- }
- }
-
- List<GroupInputSpec> groupInputSpecs =
- new ArrayList<GroupInputSpec>(FragmentSpecProto.getGroupedInputSpecsCount());
- if (FragmentSpecProto.getGroupedInputSpecsCount() > 0) {
- for (GroupInputSpecProto groupInputSpecProto : FragmentSpecProto.getGroupedInputSpecsList()) {
- groupInputSpecs.add(getGroupInputSpecFromProto(groupInputSpecProto));
- }
- }
-
- TaskSpec taskSpec =
- new TaskSpec(taskAttemptID, FragmentSpecProto.getDagName(), FragmentSpecProto.getVertexName(),
- FragmentSpecProto.getVertexParallelism(), processorDescriptor, inputSpecList,
- outputSpecList, groupInputSpecs);
- return taskSpec;
- }
-
- public static FragmentSpecProto convertTaskSpecToProto(TaskSpec taskSpec) {
- FragmentSpecProto.Builder builder = FragmentSpecProto.newBuilder();
- builder.setFragmentIdentifierString(taskSpec.getTaskAttemptID().toString());
- builder.setDagName(taskSpec.getDAGName());
- builder.setVertexName(taskSpec.getVertexName());
- builder.setVertexParallelism(taskSpec.getVertexParallelism());
- builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId());
- builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId());
-
- if (taskSpec.getProcessorDescriptor() != null) {
- builder.setProcessorDescriptor(
- convertToProto(taskSpec.getProcessorDescriptor()));
- }
-
- if (taskSpec.getInputs() != null && !taskSpec.getInputs().isEmpty()) {
- for (InputSpec inputSpec : taskSpec.getInputs()) {
- builder.addInputSpecs(convertInputSpecToProto(inputSpec));
- }
- }
-
- if (taskSpec.getOutputs() != null && !taskSpec.getOutputs().isEmpty()) {
- for (OutputSpec outputSpec : taskSpec.getOutputs()) {
- builder.addOutputSpecs(convertOutputSpecToProto(outputSpec));
- }
- }
-
- if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) {
- for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) {
- builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec));
-
- }
- }
- return builder.build();
- }
-
- private static ProcessorDescriptor convertProcessorDescriptorFromProto(
- EntityDescriptorProto proto) {
- String className = proto.getClassName();
- UserPayload payload = convertPayloadFromProto(proto);
- ProcessorDescriptor pd = ProcessorDescriptor.create(className);
- setUserPayload(pd, payload);
- return pd;
- }
-
- private static EntityDescriptorProto convertToProto(
- EntityDescriptor<?> descriptor) {
- EntityDescriptorProto.Builder builder = EntityDescriptorProto
- .newBuilder();
- builder.setClassName(descriptor.getClassName());
-
- UserPayload userPayload = descriptor.getUserPayload();
- if (userPayload != null) {
- UserPayloadProto.Builder payloadBuilder = UserPayloadProto.newBuilder();
- if (userPayload.hasPayload()) {
- payloadBuilder.setUserPayload(ByteString.copyFrom(userPayload.getPayload()));
- payloadBuilder.setVersion(userPayload.getVersion());
- }
- builder.setUserPayload(payloadBuilder.build());
- }
- if (descriptor.getHistoryText() != null) {
- try {
- builder.setHistoryText(TezCommonUtils.compressByteArrayToByteString(
- descriptor.getHistoryText().getBytes("UTF-8")));
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- }
- return builder.build();
- }
-
- private static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) {
- InputDescriptor inputDescriptor = null;
- if (inputSpecProto.hasIoDescriptor()) {
- inputDescriptor =
- convertInputDescriptorFromProto(inputSpecProto.getIoDescriptor());
- }
- InputSpec inputSpec = new InputSpec(inputSpecProto.getConnectedVertexName(), inputDescriptor,
- inputSpecProto.getPhysicalEdgeCount());
- return inputSpec;
- }
-
- private static InputDescriptor convertInputDescriptorFromProto(
- EntityDescriptorProto proto) {
- String className = proto.getClassName();
- UserPayload payload = convertPayloadFromProto(proto);
- InputDescriptor id = InputDescriptor.create(className);
- setUserPayload(id, payload);
- return id;
- }
-
- private static OutputDescriptor convertOutputDescriptorFromProto(
- EntityDescriptorProto proto) {
- String className = proto.getClassName();
- UserPayload payload = convertPayloadFromProto(proto);
- OutputDescriptor od = OutputDescriptor.create(className);
- setUserPayload(od, payload);
- return od;
- }
-
- private static IOSpecProto convertInputSpecToProto(InputSpec inputSpec) {
- IOSpecProto.Builder builder = IOSpecProto.newBuilder();
- if (inputSpec.getSourceVertexName() != null) {
- builder.setConnectedVertexName(inputSpec.getSourceVertexName());
- }
- if (inputSpec.getInputDescriptor() != null) {
- builder.setIoDescriptor(convertToProto(inputSpec.getInputDescriptor()));
- }
- builder.setPhysicalEdgeCount(inputSpec.getPhysicalEdgeCount());
- return builder.build();
- }
-
- private static OutputSpec getOutputSpecFromProto(IOSpecProto outputSpecProto) {
- OutputDescriptor outputDescriptor = null;
- if (outputSpecProto.hasIoDescriptor()) {
- outputDescriptor =
- convertOutputDescriptorFromProto(outputSpecProto.getIoDescriptor());
- }
- OutputSpec outputSpec =
- new OutputSpec(outputSpecProto.getConnectedVertexName(), outputDescriptor,
- outputSpecProto.getPhysicalEdgeCount());
- return outputSpec;
- }
-
- public static IOSpecProto convertOutputSpecToProto(OutputSpec outputSpec) {
- IOSpecProto.Builder builder = IOSpecProto.newBuilder();
- if (outputSpec.getDestinationVertexName() != null) {
- builder.setConnectedVertexName(outputSpec.getDestinationVertexName());
- }
- if (outputSpec.getOutputDescriptor() != null) {
- builder.setIoDescriptor(convertToProto(outputSpec.getOutputDescriptor()));
- }
- builder.setPhysicalEdgeCount(outputSpec.getPhysicalEdgeCount());
- return builder.build();
- }
-
- private static GroupInputSpec getGroupInputSpecFromProto(GroupInputSpecProto groupInputSpecProto) {
- GroupInputSpec groupSpec = new GroupInputSpec(groupInputSpecProto.getGroupName(),
- groupInputSpecProto.getGroupVerticesList(),
- convertInputDescriptorFromProto(groupInputSpecProto.getMergedInputDescriptor()));
- return groupSpec;
- }
-
- private static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec groupInputSpec) {
- GroupInputSpecProto.Builder builder = GroupInputSpecProto.newBuilder();
- builder.setGroupName(groupInputSpec.getGroupName());
- builder.addAllGroupVertices(groupInputSpec.getGroupVertices());
- builder.setMergedInputDescriptor(convertToProto(groupInputSpec.getMergedInputDescriptor()));
- return builder.build();
- }
-
-
- private static void setUserPayload(EntityDescriptor<?> entity, UserPayload payload) {
- if (payload != null) {
- entity.setUserPayload(payload);
- }
- }
-
- private static UserPayload convertPayloadFromProto(
- EntityDescriptorProto proto) {
- UserPayload userPayload = null;
- if (proto.hasUserPayload()) {
- if (proto.getUserPayload().hasUserPayload()) {
- userPayload =
- UserPayload.create(proto.getUserPayload().getUserPayload().asReadOnlyByteBuffer(), proto.getUserPayload().getVersion());
- } else {
- userPayload = UserPayload.create(null);
- }
- }
- return userPayload;
- }
-
- public static SourceStateProto fromVertexState(VertexState state) {
- switch (state) {
- case SUCCEEDED:
- return SourceStateProto.S_SUCCEEDED;
- case RUNNING:
- return SourceStateProto.S_RUNNING;
- default:
- throw new RuntimeException("Unexpected state: " + state);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
deleted file mode 100644
index 07703a2..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapContainerLauncher.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import org.apache.tez.serviceplugins.api.ContainerLaunchRequest;
-import org.apache.tez.serviceplugins.api.ContainerLauncher;
-import org.apache.tez.serviceplugins.api.ContainerLauncherContext;
-import org.apache.tez.serviceplugins.api.ContainerStopRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapContainerLauncher extends ContainerLauncher {
- private static final Logger LOG = LoggerFactory.getLogger(LlapContainerLauncher.class);
-
- public LlapContainerLauncher(ContainerLauncherContext containerLauncherContext) {
- super(containerLauncherContext);
- }
-
- @Override
- public void launchContainer(ContainerLaunchRequest containerLaunchRequest) {
- LOG.info("No-op launch for container: " + containerLaunchRequest.getContainerId() +
- " succeeded on host: " + containerLaunchRequest.getNodeId());
- getContext().containerLaunched(containerLaunchRequest.getContainerId());
- }
-
- @Override
- public void stopContainer(ContainerStopRequest containerStopRequest) {
- LOG.info("DEBUG: Ignoring STOP_REQUEST for event: " + containerStopRequest);
- getContext().containerStopRequested(containerStopRequest.getContainerId());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java
deleted file mode 100644
index 2884e40..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapDaemonProtocolClientProxy.java
+++ /dev/null
@@ -1,509 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import javax.net.SocketFactory;
-
-import java.io.IOException;
-import java.security.PrivilegedAction;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.Message;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
-import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemonProtocolClientImpl;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.service.AbstractService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapDaemonProtocolClientProxy extends AbstractService {
-
- private static final Logger LOG = LoggerFactory.getLogger(LlapDaemonProtocolClientProxy.class);
-
- private final ConcurrentMap<String, LlapDaemonProtocolBlockingPB> hostProxies;
-
- private final RequestManager requestManager;
- private final RetryPolicy retryPolicy;
- private final SocketFactory socketFactory;
-
- private final ListeningExecutorService requestManagerExecutor;
- private volatile ListenableFuture<Void> requestManagerFuture;
- private final Token<LlapTokenIdentifier> llapToken;
-
- public LlapDaemonProtocolClientProxy(
- int numThreads, Configuration conf, Token<LlapTokenIdentifier> llapToken) {
- super(LlapDaemonProtocolClientProxy.class.getSimpleName());
- this.hostProxies = new ConcurrentHashMap<>();
- this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
- this.llapToken = llapToken;
-
- long connectionTimeout = HiveConf.getTimeVar(conf,
- ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS);
- long retrySleep = HiveConf.getTimeVar(conf,
- ConfVars.LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MS,
- TimeUnit.MILLISECONDS);
- this.retryPolicy = RetryPolicies.retryUpToMaximumTimeWithFixedSleep(
- connectionTimeout, retrySleep, TimeUnit.MILLISECONDS);
-
- this.requestManager = new RequestManager(numThreads);
- ExecutorService localExecutor = Executors.newFixedThreadPool(1,
- new ThreadFactoryBuilder().setNameFormat("RequestManagerExecutor").build());
- this.requestManagerExecutor = MoreExecutors.listeningDecorator(localExecutor);
-
- LOG.info("Setting up taskCommunicator with" +
- "numThreads=" + numThreads +
- "retryTime(millis)=" + connectionTimeout +
- "retrySleep(millis)=" + retrySleep);
- }
-
- @Override
- public void serviceStart() {
- requestManagerFuture = requestManagerExecutor.submit(requestManager);
- Futures.addCallback(requestManagerFuture, new FutureCallback<Void>() {
- @Override
- public void onSuccess(Void result) {
- LOG.info("RequestManager shutdown");
- }
-
- @Override
- public void onFailure(Throwable t) {
- LOG.warn("RequestManager shutdown with error", t);
- }
- });
- }
-
- @Override
- public void serviceStop() {
- if (requestManagerFuture != null) {
- requestManager.shutdown();
- requestManagerFuture.cancel(true);
- }
- requestManagerExecutor.shutdown();
- }
-
- public void sendSubmitWork(SubmitWorkRequestProto request, String host, int port,
- final ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
- LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
- requestManager.queueRequest(new SubmitWorkCallable(nodeId, request, callback));
- }
-
- public void sendSourceStateUpdate(final SourceStateUpdatedRequestProto request, final String host,
- final int port,
- final ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
- LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
- requestManager.queueRequest(
- new SendSourceStateUpdateCallable(nodeId, request, callback));
- }
-
- public void sendQueryComplete(final QueryCompleteRequestProto request, final String host,
- final int port,
- final ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
- LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
- requestManager.queueRequest(new SendQueryCompleteCallable(nodeId, request, callback));
- }
-
- public void sendTerminateFragment(final TerminateFragmentRequestProto request, final String host,
- final int port,
- final ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
- LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
- requestManager.queueRequest(new SendTerminateFragmentCallable(nodeId, request, callback));
- }
-
- @VisibleForTesting
- static class RequestManager implements Callable<Void> {
-
- private final Lock lock = new ReentrantLock();
- private final AtomicBoolean isShutdown = new AtomicBoolean(false);
- private final Condition queueCondition = lock.newCondition();
- private final AtomicBoolean shouldRun = new AtomicBoolean(false);
-
- private final int maxConcurrentRequestsPerNode = 1;
- private final ListeningExecutorService executor;
-
-
- // Tracks new additions via add, while the loop is processing existing ones.
- private final LinkedList<CallableRequest> newRequestList = new LinkedList<>();
-
- // Tracks existing requests which are cycled through.
- private final LinkedList<CallableRequest> pendingRequests = new LinkedList<>();
-
- // Tracks requests executing per node
- private final ConcurrentMap<LlapNodeId, AtomicInteger> runningRequests = new ConcurrentHashMap<>();
-
- // Tracks completed requests pre node
- private final LinkedList<LlapNodeId> completedNodes = new LinkedList<>();
-
- public RequestManager(int numThreads) {
- ExecutorService localExecutor = Executors.newFixedThreadPool(numThreads,
- new ThreadFactoryBuilder().setNameFormat("TaskCommunicator #%2d").build());
- executor = MoreExecutors.listeningDecorator(localExecutor);
- }
-
-
- @VisibleForTesting
- Set<LlapNodeId> currentLoopDisabledNodes = new HashSet<>();
- @VisibleForTesting
- List<CallableRequest> currentLoopSkippedRequests = new LinkedList<>();
- @Override
- public Void call() {
- // Caches disabled nodes for quicker lookups and ensures a request on a node which was skipped
- // does not go out of order.
- while (!isShutdown.get()) {
- lock.lock();
- try {
- while (!shouldRun.get()) {
- queueCondition.await();
- break; // Break out and try executing.
- }
- boolean shouldBreak = process();
- if (shouldBreak) {
- break;
- }
- } catch (InterruptedException e) {
- if (isShutdown.get()) {
- break;
- } else {
- LOG.warn("RunLoop interrupted without being shutdown first");
- throw new RuntimeException(e);
- }
- } finally {
- lock.unlock();
- }
- }
- LOG.info("CallScheduler loop exiting");
- return null;
- }
-
- /* Add a new request to be executed */
- public void queueRequest(CallableRequest request) {
- synchronized (newRequestList) {
- newRequestList.add(request);
- shouldRun.set(true);
- }
- notifyRunLoop();
- }
-
- /* Indicates a request has completed on a node */
- public void requestFinished(LlapNodeId nodeId) {
- synchronized (completedNodes) {
- completedNodes.add(nodeId);
- shouldRun.set(true);
- }
- notifyRunLoop();
- }
-
- public void shutdown() {
- if (!isShutdown.getAndSet(true)) {
- executor.shutdownNow();
- notifyRunLoop();
- }
- }
-
- @VisibleForTesting
- void submitToExecutor(CallableRequest request, LlapNodeId nodeId) {
- ListenableFuture<SourceStateUpdatedResponseProto> future =
- executor.submit(request);
- Futures.addCallback(future, new ResponseCallback(request.getCallback(), nodeId, this));
- }
-
- @VisibleForTesting
- boolean process() {
- if (isShutdown.get()) {
- return true;
- }
- currentLoopDisabledNodes.clear();
- currentLoopSkippedRequests.clear();
-
- // Set to false to block the next loop. This must be called before draining the lists,
- // otherwise an add/completion after draining the lists but before setting it to false,
- // will not trigger a run. May cause one unnecessary run if an add comes in before drain.
- // drain list. add request (setTrue). setFalse needs to be avoided.
- shouldRun.compareAndSet(true, false);
- // Drain any calls which may have come in during the last execution of the loop.
- drainNewRequestList(); // Locks newRequestList
- drainCompletedNodes(); // Locks completedNodes
-
-
- Iterator<CallableRequest> iterator = pendingRequests.iterator();
- while (iterator.hasNext()) {
- CallableRequest request = iterator.next();
- iterator.remove();
- LlapNodeId nodeId = request.getNodeId();
- if (canRunForNode(nodeId, currentLoopDisabledNodes)) {
- submitToExecutor(request, nodeId);
- } else {
- currentLoopDisabledNodes.add(nodeId);
- currentLoopSkippedRequests.add(request);
- }
- }
- // Tried scheduling everything that could be scheduled in this loop.
- pendingRequests.addAll(0, currentLoopSkippedRequests);
- return false;
- }
-
- private void drainNewRequestList() {
- synchronized (newRequestList) {
- if (!newRequestList.isEmpty()) {
- pendingRequests.addAll(newRequestList);
- newRequestList.clear();
- }
- }
- }
-
- private void drainCompletedNodes() {
- synchronized (completedNodes) {
- if (!completedNodes.isEmpty()) {
- for (LlapNodeId nodeId : completedNodes) {
- runningRequests.get(nodeId).decrementAndGet();
- }
- }
- completedNodes.clear();
- }
- }
-
- private boolean canRunForNode(LlapNodeId nodeId, Set<LlapNodeId> currentRunDisabledNodes) {
- if (currentRunDisabledNodes.contains(nodeId)) {
- return false;
- } else {
- AtomicInteger count = runningRequests.get(nodeId);
- if (count == null) {
- count = new AtomicInteger(0);
- AtomicInteger old = runningRequests.putIfAbsent(nodeId, count);
- count = old != null ? old : count;
- }
- if (count.incrementAndGet() <= maxConcurrentRequestsPerNode) {
- return true;
- } else {
- count.decrementAndGet();
- return false;
- }
- }
- }
-
- private void notifyRunLoop() {
- lock.lock();
- try {
- queueCondition.signal();
- } finally {
- lock.unlock();
- }
- }
- }
-
-
- private static final class ResponseCallback<TYPE extends Message>
- implements FutureCallback<TYPE> {
-
- private final ExecuteRequestCallback<TYPE> callback;
- private final LlapNodeId nodeId;
- private final RequestManager requestManager;
-
- public ResponseCallback(ExecuteRequestCallback<TYPE> callback, LlapNodeId nodeId,
- RequestManager requestManager) {
- this.callback = callback;
- this.nodeId = nodeId;
- this.requestManager = requestManager;
- }
-
- @Override
- public void onSuccess(TYPE result) {
- try {
- callback.setResponse(result);
- } finally {
- requestManager.requestFinished(nodeId);
- }
- }
-
- @Override
- public void onFailure(Throwable t) {
- try {
- callback.indicateError(t);
- } finally {
- requestManager.requestFinished(nodeId);
- }
- }
- }
-
- @VisibleForTesting
- static abstract class CallableRequest<REQUEST extends Message, RESPONSE extends Message>
- implements Callable {
-
- final LlapNodeId nodeId;
- final ExecuteRequestCallback<RESPONSE> callback;
- final REQUEST request;
-
-
- protected CallableRequest(LlapNodeId nodeId, REQUEST request, ExecuteRequestCallback<RESPONSE> callback) {
- this.nodeId = nodeId;
- this.request = request;
- this.callback = callback;
- }
-
- public LlapNodeId getNodeId() {
- return nodeId;
- }
-
- public ExecuteRequestCallback<RESPONSE> getCallback() {
- return callback;
- }
-
- public abstract RESPONSE call() throws Exception;
- }
-
- private class SubmitWorkCallable extends CallableRequest<SubmitWorkRequestProto, SubmitWorkResponseProto> {
-
- protected SubmitWorkCallable(LlapNodeId nodeId,
- SubmitWorkRequestProto submitWorkRequestProto,
- ExecuteRequestCallback<SubmitWorkResponseProto> callback) {
- super(nodeId, submitWorkRequestProto, callback);
- }
-
- @Override
- public SubmitWorkResponseProto call() throws Exception {
- return getProxy(nodeId).submitWork(null, request);
- }
- }
-
- private class SendSourceStateUpdateCallable
- extends CallableRequest<SourceStateUpdatedRequestProto, SourceStateUpdatedResponseProto> {
-
- public SendSourceStateUpdateCallable(LlapNodeId nodeId,
- SourceStateUpdatedRequestProto request,
- ExecuteRequestCallback<SourceStateUpdatedResponseProto> callback) {
- super(nodeId, request, callback);
- }
-
- @Override
- public SourceStateUpdatedResponseProto call() throws Exception {
- return getProxy(nodeId).sourceStateUpdated(null, request);
- }
- }
-
- private class SendQueryCompleteCallable
- extends CallableRequest<QueryCompleteRequestProto, QueryCompleteResponseProto> {
-
- protected SendQueryCompleteCallable(LlapNodeId nodeId,
- QueryCompleteRequestProto queryCompleteRequestProto,
- ExecuteRequestCallback<QueryCompleteResponseProto> callback) {
- super(nodeId, queryCompleteRequestProto, callback);
- }
-
- @Override
- public QueryCompleteResponseProto call() throws Exception {
- return getProxy(nodeId).queryComplete(null, request);
- }
- }
-
- private class SendTerminateFragmentCallable
- extends CallableRequest<TerminateFragmentRequestProto, TerminateFragmentResponseProto> {
-
- protected SendTerminateFragmentCallable(LlapNodeId nodeId,
- TerminateFragmentRequestProto terminateFragmentRequestProto,
- ExecuteRequestCallback<TerminateFragmentResponseProto> callback) {
- super(nodeId, terminateFragmentRequestProto, callback);
- }
-
- @Override
- public TerminateFragmentResponseProto call() throws Exception {
- return getProxy(nodeId).terminateFragment(null, request);
- }
- }
-
- public interface ExecuteRequestCallback<T extends Message> {
- void setResponse(T response);
- void indicateError(Throwable t);
- }
-
- private LlapDaemonProtocolBlockingPB getProxy(final LlapNodeId nodeId) {
- String hostId = getHostIdentifier(nodeId.getHostname(), nodeId.getPort());
-
- LlapDaemonProtocolBlockingPB proxy = hostProxies.get(hostId);
- if (proxy == null) {
- if (llapToken == null) {
- proxy = new LlapDaemonProtocolClientImpl(getConfig(), nodeId.getHostname(),
- nodeId.getPort(), retryPolicy, socketFactory);
- } else {
- UserGroupInformation ugi;
- try {
- ugi = UserGroupInformation.getCurrentUser();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- Token<LlapTokenIdentifier> nodeToken = new Token<LlapTokenIdentifier>(llapToken);
- SecurityUtil.setTokenService(nodeToken, NetUtils.createSocketAddrForHost(
- nodeId.getHostname(), nodeId.getPort()));
- ugi.addToken(nodeToken);
- proxy = ugi.doAs(new PrivilegedAction<LlapDaemonProtocolBlockingPB>() {
- @Override
- public LlapDaemonProtocolBlockingPB run() {
- return new LlapDaemonProtocolClientImpl(getConfig(), nodeId.getHostname(),
- nodeId.getPort(), retryPolicy, socketFactory);
- }
- });
- }
-
- LlapDaemonProtocolBlockingPB proxyOld = hostProxies.putIfAbsent(hostId, proxy);
- if (proxyOld != null) {
- // TODO Shutdown the new proxy.
- proxy = proxyOld;
- }
- }
- return proxy;
- }
-
- private String getHostIdentifier(String hostname, int port) {
- return hostname + ":" + port;
- }
-}
[04/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java b/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
deleted file mode 100644
index 6beb4f8..0000000
--- a/llap-server/src/java/org/apache/tez/dag/app/rm/LlapTaskSchedulerService.java
+++ /dev/null
@@ -1,1512 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.NavigableMap;
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import org.apache.commons.lang.mutable.MutableInt;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.registry.ServiceInstance;
-import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
-import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.util.Clock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskScheduler;
-import org.apache.tez.serviceplugins.api.TaskSchedulerContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapTaskSchedulerService extends TaskScheduler {
-
- private static final Logger LOG = LoggerFactory.getLogger(LlapTaskSchedulerService.class);
-
- private final Configuration conf;
-
- // interface into the registry service
- private ServiceInstanceSet activeInstances;
-
- // Tracks all instances, including ones which have been disabled in the past.
- // LinkedHashMap to provide the same iteration order when selecting a random host.
- @VisibleForTesting
- final Map<ServiceInstance, NodeInfo> instanceToNodeMap = new LinkedHashMap<>();
- // TODO Ideally, remove elements from this once it's known that no tasks are linked to the instance (all deallocated)
-
- // Tracks tasks which could not be allocated immediately.
- @VisibleForTesting
- final TreeMap<Priority, List<TaskInfo>> pendingTasks = new TreeMap<>(new Comparator<Priority>() {
- @Override
- public int compare(Priority o1, Priority o2) {
- return o1.getPriority() - o2.getPriority();
- }
- });
-
- // Tracks running and queued tasks. Cleared after a task completes.
- private final ConcurrentMap<Object, TaskInfo> knownTasks = new ConcurrentHashMap<>();
- private final TreeMap<Integer, TreeSet<TaskInfo>> runningTasks = new TreeMap<>();
- private static final TaskStartComparator TASK_INFO_COMPARATOR = new TaskStartComparator();
-
- // Queue for disabled nodes. Nodes make it out of this queue when their expiration timeout is hit.
- @VisibleForTesting
- final DelayQueue<NodeInfo> disabledNodesQueue = new DelayQueue<>();
-
- private final boolean forceLocation;
-
- private final ContainerFactory containerFactory;
- private final Random random = new Random();
- private final Clock clock;
-
- private final ListeningExecutorService nodeEnabledExecutor;
- private final NodeEnablerCallable nodeEnablerCallable =
- new NodeEnablerCallable();
-
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
- private final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock();
-
- private final Lock scheduleLock = new ReentrantLock();
- private final Condition scheduleCondition = scheduleLock.newCondition();
- private final AtomicBoolean pendingScheduleInvodations = new AtomicBoolean(false);
- private final ListeningExecutorService schedulerExecutor;
- private final SchedulerCallable schedulerCallable = new SchedulerCallable();
-
- private final AtomicBoolean isStopped = new AtomicBoolean(false);
- // Tracks total pending preemptions.
- private final AtomicInteger pendingPreemptions = new AtomicInteger(0);
- // Tracks pending preemptions per host, using the hostname || Always to be accessed inside a lock
- private final Map<String, MutableInt> pendingPreemptionsPerHost = new HashMap<>();
-
- private final NodeBlacklistConf nodeBlacklistConf;
-
- // Per daemon
- private final int memoryPerInstance;
- private final int coresPerInstance;
- private final int executorsPerInstance;
-
- private final int numSchedulableTasksPerNode;
-
- // Per Executor Thread
- private final Resource resourcePerExecutor;
-
- private final LlapRegistryService registry = new LlapRegistryService(false);
-
- private volatile ListenableFuture<Void> nodeEnablerFuture;
- private volatile ListenableFuture<Void> schedulerFuture;
-
- @VisibleForTesting
- private final AtomicInteger dagCounter = new AtomicInteger(1);
- // Statistics to track allocations
- // All of stats variables are visible for testing.
- @VisibleForTesting
- StatsPerDag dagStats = new StatsPerDag();
-
- public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) {
- this(taskSchedulerContext, new SystemClock());
- }
-
- @VisibleForTesting
- public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock) {
- super(taskSchedulerContext);
- this.clock = clock;
- try {
- this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload());
- } catch (IOException e) {
- throw new TezUncheckedException(
- "Failed to parse user payload for " + LlapTaskSchedulerService.class.getSimpleName(), e);
- }
- this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(),
- taskSchedulerContext.getCustomClusterIdentifier());
- this.memoryPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB);
- this.coresPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_VCPUS_PER_INSTANCE);
- this.executorsPerInstance = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
- this.nodeBlacklistConf = new NodeBlacklistConf(
- HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS,
- TimeUnit.MILLISECONDS),
- HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MAX_TIMEOUT_MS,
- TimeUnit.MILLISECONDS),
- HiveConf.getFloatVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_DISABLE_BACK_OFF_FACTOR));
-
- this.numSchedulableTasksPerNode =
- HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NUM_SCHEDULABLE_TASKS_PER_NODE);
-
- long localityDelayMs = HiveConf
- .getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY, TimeUnit.MILLISECONDS);
- if (localityDelayMs == -1) {
- this.forceLocation = true;
- } else {
- this.forceLocation = false;
- }
-
- int memoryPerExecutor = (int) (memoryPerInstance / (float) executorsPerInstance);
- int coresPerExecutor = (int) (coresPerInstance / (float) executorsPerInstance);
- this.resourcePerExecutor = Resource.newInstance(memoryPerExecutor, coresPerExecutor);
-
- String instanceId = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS);
-
- Preconditions.checkNotNull(instanceId, ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname
- + " must be defined");
-
- ExecutorService executorServiceRaw =
- Executors.newFixedThreadPool(1,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapSchedulerNodeEnabler").build());
- nodeEnabledExecutor = MoreExecutors.listeningDecorator(executorServiceRaw);
-
- ExecutorService schedulerExecutorServiceRaw = Executors.newFixedThreadPool(1,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build());
- schedulerExecutor = MoreExecutors.listeningDecorator(schedulerExecutorServiceRaw);
-
- LOG.info("Running with configuration: " + "memoryPerInstance=" + memoryPerInstance
- + ", vCoresPerInstance=" + coresPerInstance + ", executorsPerInstance="
- + executorsPerInstance + ", resourcePerInstanceInferred=" + resourcePerExecutor
- + ", nodeBlacklistConf=" + nodeBlacklistConf
- + ", forceLocation=" + forceLocation);
- }
-
- @Override
- public void initialize() {
- registry.init(conf);
- }
-
- @Override
- public void start() throws IOException {
- writeLock.lock();
- try {
- nodeEnablerFuture = nodeEnabledExecutor.submit(nodeEnablerCallable);
- Futures.addCallback(nodeEnablerFuture, new FutureCallback<Void>() {
- @Override
- public void onSuccess(Void result) {
- LOG.info("NodeEnabledThread exited");
- }
-
- @Override
- public void onFailure(Throwable t) {
- LOG.warn("NodeEnabledThread exited with error", t);
- }
- });
- schedulerFuture = schedulerExecutor.submit(schedulerCallable);
- Futures.addCallback(schedulerFuture, new FutureCallback<Void>() {
- @Override
- public void onSuccess(Void result) {
- LOG.info("SchedulerThread exited");
- }
-
- @Override
- public void onFailure(Throwable t) {
- LOG.warn("SchedulerThread exited with error", t);
- }
- });
- registry.start();
- activeInstances = registry.getInstances();
- for (ServiceInstance inst : activeInstances.getAll().values()) {
- addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- @Override
- public void shutdown() {
- writeLock.lock();
- try {
- if (!this.isStopped.getAndSet(true)) {
- nodeEnablerCallable.shutdown();
- if (nodeEnablerFuture != null) {
- nodeEnablerFuture.cancel(true);
- }
- nodeEnabledExecutor.shutdownNow();
-
- schedulerCallable.shutdown();
- if (schedulerFuture != null) {
- schedulerFuture.cancel(true);
- }
- schedulerExecutor.shutdownNow();
-
- if (registry != null) {
- registry.stop();
- }
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- @Override
- public Resource getTotalResources() {
- int memory = 0;
- int vcores = 0;
- readLock.lock();
- try {
- for (ServiceInstance inst : activeInstances.getAll().values()) {
- if (inst.isAlive()) {
- Resource r = inst.getResource();
- LOG.info("Found instance " + inst);
- memory += r.getMemory();
- vcores += r.getVirtualCores();
- } else {
- LOG.info("Ignoring dead instance " + inst);
- }
- }
- } finally {
- readLock.unlock();
- }
-
- return Resource.newInstance(memory, vcores);
- }
-
- /**
- * The difference between this and getTotalResources() is that this only gives currently free
- * resource instances, while the other lists all the instances that may become available in a
- * while.
- */
- @Override
- public Resource getAvailableResources() {
- // need a state store eventually for current state & measure backoffs
- int memory = 0;
- int vcores = 0;
- readLock.lock();
- try {
- for (Entry<ServiceInstance, NodeInfo> entry : instanceToNodeMap.entrySet()) {
- if (entry.getKey().isAlive() && !entry.getValue().isDisabled()) {
- Resource r = entry.getKey().getResource();
- memory += r.getMemory();
- vcores += r.getVirtualCores();
- }
- }
- } finally {
- readLock.unlock();
- }
-
- return Resource.newInstance(memory, vcores);
- }
-
- @Override
- public int getClusterNodeCount() {
- readLock.lock();
- try {
- int n = 0;
- for (ServiceInstance inst : activeInstances.getAll().values()) {
- if (inst.isAlive()) {
- n++;
- }
- }
- return n;
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public void dagComplete() {
- // This is effectively DAG completed, and can be used to reset statistics being tracked.
- LOG.info("DAG: " + dagCounter.get() + " completed. Scheduling stats: " + dagStats);
- dagCounter.incrementAndGet();
- dagStats = new StatsPerDag();
- }
-
- @Override
- public void blacklistNode(NodeId nodeId) {
- LOG.info("BlacklistNode not supported");
- }
-
- @Override
- public void unblacklistNode(NodeId nodeId) {
- LOG.info("unBlacklistNode not supported");
- }
-
- @Override
- public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks,
- Priority priority, Object containerSignature, Object clientCookie) {
- TaskInfo taskInfo =
- new TaskInfo(task, clientCookie, priority, capability, hosts, racks, clock.getTime());
- writeLock.lock();
- try {
- dagStats.registerTaskRequest(hosts, racks);
- } finally {
- writeLock.unlock();
- }
- addPendingTask(taskInfo);
- trySchedulingPendingTasks();
- }
-
- @Override
- public void allocateTask(Object task, Resource capability, ContainerId containerId,
- Priority priority, Object containerSignature, Object clientCookie) {
- // Container affinity can be implemented as Host affinity for LLAP. Not required until
- // 1:1 edges are used in Hive.
- TaskInfo taskInfo =
- new TaskInfo(task, clientCookie, priority, capability, null, null, clock.getTime());
- writeLock.lock();
- try {
- dagStats.registerTaskRequest(null, null);
- } finally {
- writeLock.unlock();
- }
- addPendingTask(taskInfo);
- trySchedulingPendingTasks();
- }
-
-
- // This may be invoked before a container is ever assigned to a task. allocateTask... app decides
- // the task is no longer required, and asks for a de-allocation.
- @Override
- public boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason, String diagnostics) {
- writeLock.lock(); // Updating several local structures
- TaskInfo taskInfo;
- try {
- taskInfo = unregisterTask(task);
- if (taskInfo == null) {
- LOG.error("Could not determine ContainerId for task: "
- + task
- + " . Could have hit a race condition. Ignoring."
- + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
- return false;
- }
- if (taskInfo.containerId == null) {
- if (taskInfo.assigned) {
- LOG.error("Task: "
- + task
- + " assigned, but could not find the corresponding containerId."
- + " The query may hang since this \"unknown\" container is now taking up a slot permanently");
- } else {
- LOG.info("Ignoring deallocate request for task " + task
- + " which hasn't been assigned to a container");
- removePendingTask(taskInfo);
- }
- return false;
- }
- ServiceInstance assignedInstance = taskInfo.assignedInstance;
- assert assignedInstance != null;
-
- NodeInfo nodeInfo = instanceToNodeMap.get(assignedInstance);
- assert nodeInfo != null;
-
- // Re-enable the node if preempted
- if (taskInfo.preempted) {
- LOG.info("Processing deallocateTask for {} which was preempted, EndReason={}", task, endReason);
- unregisterPendingPreemption(taskInfo.assignedInstance.getHost());
- nodeInfo.registerUnsuccessfulTaskEnd(true);
- if (nodeInfo.isDisabled()) {
- // Re-enable the node. If a task succeeded, a slot may have become available.
- // Also reset commFailures since a task was able to communicate back and indicate success.
- nodeInfo.enableNode();
- // Re-insert into the queue to force the poll thread to remove the element.
- if ( disabledNodesQueue.remove(nodeInfo)) {
- disabledNodesQueue.add(nodeInfo);
- }
- }
- // In case of success, trigger a scheduling run for pending tasks.
- trySchedulingPendingTasks();
- } else {
- if (taskSucceeded) {
- // The node may have been blacklisted at this point - which means it may not be in the
- // activeNodeList.
-
- nodeInfo.registerTaskSuccess();
-
- if (nodeInfo.isDisabled()) {
- // Re-enable the node. If a task succeeded, a slot may have become available.
- // Also reset commFailures since a task was able to communicate back and indicate success.
- nodeInfo.enableNode();
- // Re-insert into the queue to force the poll thread to remove the element.
- if (disabledNodesQueue.remove(nodeInfo)) {
- disabledNodesQueue.add(nodeInfo);
- }
- }
- // In case of success, trigger a scheduling run for pending tasks.
- trySchedulingPendingTasks();
-
- } else if (!taskSucceeded) {
- nodeInfo.registerUnsuccessfulTaskEnd(false);
- if (endReason != null && EnumSet
- .of(TaskAttemptEndReason.EXECUTOR_BUSY, TaskAttemptEndReason.COMMUNICATION_ERROR)
- .contains(endReason)) {
- if (endReason == TaskAttemptEndReason.COMMUNICATION_ERROR) {
- dagStats.registerCommFailure(taskInfo.assignedInstance.getHost());
- } else if (endReason == TaskAttemptEndReason.EXECUTOR_BUSY) {
- dagStats.registerTaskRejected(taskInfo.assignedInstance.getHost());
- }
- }
- boolean commFailure =
- endReason != null && endReason == TaskAttemptEndReason.COMMUNICATION_ERROR;
- disableInstance(assignedInstance, commFailure);
- }
- }
- } finally {
- writeLock.unlock();
- }
- getContext().containerBeingReleased(taskInfo.containerId);
- return true;
- }
-
- @Override
- public Object deallocateContainer(ContainerId containerId) {
- LOG.debug("Ignoring deallocateContainer for containerId: " + containerId);
- // Containers are not being tracked for re-use.
- // This is safe to ignore since a deallocate task will come in.
- return null;
- }
-
- @Override
- public void setShouldUnregister() {
-
- }
-
- @Override
- public boolean hasUnregistered() {
- // Nothing to do. No registration involved.
- return true;
- }
-
- private ExecutorService createAppCallbackExecutorService() {
- return Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
- .setNameFormat("TaskSchedulerAppCaller #%d").setDaemon(true).build());
- }
-
- /**
- * @param request the list of preferred hosts. null implies any host
- * @return
- */
- private SelectHostResult selectHost(TaskInfo request) {
- String[] requestedHosts = request.requestedHosts;
- readLock.lock(); // Read-lock. Not updating any stats at the moment.
- try {
- // Check if any hosts are active.
- if (getAvailableResources().getMemory() <= 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Refreshing instances since total memory is 0");
- }
- refreshInstances();
- }
-
- // If there's no memory available, fail
- if (getTotalResources().getMemory() <= 0) {
- return SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY;
- }
-
- if (requestedHosts != null && requestedHosts.length > 0) {
- int prefHostCount = -1;
- boolean requestedHostExists = false;
- for (String host : requestedHosts) {
- prefHostCount++;
- // Pick the first host always. Weak attempt at cache affinity.
- Set<ServiceInstance> instances = activeInstances.getByHost(host);
- if (!instances.isEmpty()) {
- requestedHostExists = true;
- for (ServiceInstance inst : instances) {
- NodeInfo nodeInfo = instanceToNodeMap.get(inst);
- if (nodeInfo != null && nodeInfo.canAcceptTask()) {
- LOG.info("Assigning " + inst + " when looking for " + host + "." +
- " FirstRequestedHost=" + (prefHostCount == 0) +
- (requestedHosts.length > 1 ? "#prefLocations=" + requestedHosts.length : ""));
- return new SelectHostResult(inst, nodeInfo);
- }
- }
- }
- }
- // Check if forcing the location is required.
- if (forceLocation) {
- if (requestedHostExists) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping non-local location allocation for [" + request.task +
- "] when trying to allocate on [" + Arrays.toString(requestedHosts) + "]");
- }
- return SELECT_HOST_RESULT_DELAYED_LOCALITY;
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Not skipping non-local location allocation for [" + request.task +
- "] when trying to allocate on [" + Arrays.toString(requestedHosts) +
- "] since none of these hosts are part of the known list");
- }
- }
- }
- }
- /* fall through - miss in locality (random scheduling) */
- Entry<ServiceInstance, NodeInfo>[] all =
- instanceToNodeMap.entrySet().toArray(new Entry[instanceToNodeMap.size()]);
- // Check again
- if (all.length > 0) {
- int n = random.nextInt(all.length);
- // start at random offset and iterate whole list
- for (int i = 0; i < all.length; i++) {
- Entry<ServiceInstance, NodeInfo> inst = all[(i + n) % all.length];
- if (inst.getValue().canAcceptTask()) {
- LOG.info("Assigning " + inst + " when looking for any host, from #hosts=" + all.length +
- ", requestedHosts=" +
- ((requestedHosts == null || requestedHosts.length == 0) ? "null" :
- Arrays.toString(requestedHosts)));
- return new SelectHostResult(inst.getKey(), inst.getValue());
- }
- }
- }
- return SELECT_HOST_RESULT_DELAYED_RESOURCES;
- } finally {
- readLock.unlock();
- }
- }
-
- // TODO Each refresh operation should addNodes if they don't already exist.
- // Even better would be to get notifications from the service impl when a node gets added or removed.
- // Instead of having to walk through the entire list. The computation of a node getting added or
- // removed already exists in the DynamicRegistry implementation.
- private void refreshInstances() {
- try {
- activeInstances.refresh(); // handles its own sync
- } catch (IOException ioe) {
- LOG.warn("Could not refresh list of active instances", ioe);
- }
- }
-
- private void scanForNodeChanges() {
- /* check again whether nodes are disabled or just missing */
- writeLock.lock();
- try {
- for (ServiceInstance inst : activeInstances.getAll().values()) {
- if (inst.isAlive() && instanceToNodeMap.containsKey(inst) == false) {
- /* that's a good node, not added to the allocations yet */
- LOG.info("Found a new node: " + inst + ".");
- addNode(inst, new NodeInfo(inst, nodeBlacklistConf, clock, numSchedulableTasksPerNode));
- }
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- private void addNode(ServiceInstance inst, NodeInfo node) {
- LOG.info("Adding node: " + inst);
- instanceToNodeMap.put(inst, node);
- // Trigger scheduling since a new node became available.
- trySchedulingPendingTasks();
- }
-
- private void reenableDisabledNode(NodeInfo nodeInfo) {
- writeLock.lock();
- try {
- if (nodeInfo.hadCommFailure()) {
- // If the node being re-enabled was not marked busy previously, then it was disabled due to
- // some other failure. Refresh the service list to see if it's been removed permanently.
- refreshInstances();
- }
- LOG.info("Attempting to re-enable node: " + nodeInfo.getServiceInstance());
- if (nodeInfo.getServiceInstance().isAlive()) {
- nodeInfo.enableNode();
- } else {
- if (LOG.isInfoEnabled()) {
- LOG.info("Removing dead node " + nodeInfo);
- }
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- private void disableInstance(ServiceInstance instance, boolean isCommFailure) {
- writeLock.lock();
- try {
- NodeInfo nodeInfo = instanceToNodeMap.get(instance);
- if (nodeInfo == null || nodeInfo.isDisabled()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Node: " + instance + " already disabled, or invalid. Not doing anything.");
- }
- } else {
- nodeInfo.disableNode(isCommFailure);
- // TODO: handle task to container map events in case of hard failures
- disabledNodesQueue.add(nodeInfo);
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- private void addPendingTask(TaskInfo taskInfo) {
- writeLock.lock();
- try {
- List<TaskInfo> tasksAtPriority = pendingTasks.get(taskInfo.priority);
- if (tasksAtPriority == null) {
- tasksAtPriority = new LinkedList<>();
- pendingTasks.put(taskInfo.priority, tasksAtPriority);
- }
- tasksAtPriority.add(taskInfo);
- knownTasks.putIfAbsent(taskInfo.task, taskInfo);
- } finally {
- writeLock.unlock();
- }
- }
-
- /* Remove a task from the pending list */
- private void removePendingTask(TaskInfo taskInfo) {
- writeLock.lock();
- try {
- Priority priority = taskInfo.priority;
- List<TaskInfo> taskInfoList = pendingTasks.get(priority);
- if (taskInfoList == null || taskInfoList.isEmpty() || !taskInfoList.remove(taskInfo)) {
- LOG.warn("Could not find task: " + taskInfo.task + " in pending list, at priority: "
- + priority);
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- /* Register a running task into the runningTasks structure */
- private void registerRunningTask(TaskInfo taskInfo) {
- writeLock.lock();
- try {
- int priority = taskInfo.priority.getPriority();
- TreeSet<TaskInfo> tasksAtpriority = runningTasks.get(priority);
- if (tasksAtpriority == null) {
- tasksAtpriority = new TreeSet<>(TASK_INFO_COMPARATOR);
- runningTasks.put(priority, tasksAtpriority);
- }
- tasksAtpriority.add(taskInfo);
- } finally {
- writeLock.unlock();
- }
- }
-
- /* Unregister a task from the known and running structures */
- private TaskInfo unregisterTask(Object task) {
- writeLock.lock();
- try {
- TaskInfo taskInfo = knownTasks.remove(task);
- if (taskInfo != null) {
- if (taskInfo.assigned) {
- // Remove from the running list.
- int priority = taskInfo.priority.getPriority();
- Set<TaskInfo> tasksAtPriority = runningTasks.get(priority);
- Preconditions.checkState(tasksAtPriority != null,
- "runningTasks should contain an entry if the task was in running state. Caused by task: {}", task);
- tasksAtPriority.remove(taskInfo);
- if (tasksAtPriority.isEmpty()) {
- runningTasks.remove(priority);
- }
- }
- } else {
- LOG.warn("Could not find TaskInfo for task: {}. Not removing it from the running set", task);
- }
- return taskInfo;
- } finally {
- writeLock.unlock();
- }
- }
-
- private enum ScheduleResult {
- // Successfully scheduled
- SCHEDULED,
-
- // Delayed to find a local match
- DELAYED_LOCALITY,
-
- // Delayed due to temporary resource availability
- DELAYED_RESOURCES,
-
- // Inadequate total resources - will never succeed / wait for new executors to become available
- INADEQUATE_TOTAL_RESOURCES,
- }
-
- @VisibleForTesting
- protected void schedulePendingTasks() {
- writeLock.lock();
- try {
- Iterator<Entry<Priority, List<TaskInfo>>> pendingIterator =
- pendingTasks.entrySet().iterator();
- while (pendingIterator.hasNext()) {
- Entry<Priority, List<TaskInfo>> entry = pendingIterator.next();
- List<TaskInfo> taskListAtPriority = entry.getValue();
- Iterator<TaskInfo> taskIter = taskListAtPriority.iterator();
- boolean scheduledAllAtPriority = true;
- while (taskIter.hasNext()) {
-
- // TODO Optimization: Add a check to see if there's any capacity available. No point in
- // walking through all active nodes, if they don't have potential capacity.
-
- TaskInfo taskInfo = taskIter.next();
- if (taskInfo.getNumPreviousAssignAttempts() == 1) {
- dagStats.registerDelayedAllocation();
- }
- taskInfo.triedAssigningTask();
- ScheduleResult scheduleResult = scheduleTask(taskInfo);
- if (scheduleResult == ScheduleResult.SCHEDULED) {
- taskIter.remove();
- } else {
- // TODO Handle INADEQUATE_TOTAL_RESOURCES eventually - either by throwin an error immediately,
- // or waiting for some timeout for new executors and then throwing an error
-
- // Try pre-empting a task so that a higher priority task can take it's place.
- // Preempt only if there's no pending preemptions to avoid preempting twice for a task.
- String[] potentialHosts;
- if (scheduleResult == ScheduleResult.DELAYED_LOCALITY) {
- // preempt only on specific hosts, if no preemptions already exist on those.
- potentialHosts = taskInfo.requestedHosts;
- //Protect against a bad location being requested.
- if (potentialHosts == null || potentialHosts.length == 0) {
- potentialHosts = null;
- }
- } else {
- // preempt on any host.
- potentialHosts = null;
- }
-
- if (potentialHosts != null) {
- // Preempt on specific host
- boolean shouldPreempt = true;
- for (String host : potentialHosts) {
- // Preempt only if there are not pending preemptions on the same host
- // When the premption registers, the request at the highest priority will be given the slot,
- // even if the initial request was for some other task.
- // TODO Maybe register which task the preemption was for, to avoid a bad non-local allocation.
- MutableInt pendingHostPreemptions = pendingPreemptionsPerHost.get(host);
- if (pendingHostPreemptions != null && pendingHostPreemptions.intValue() > 0) {
- shouldPreempt = false;
- break;
- }
- }
- if (shouldPreempt) {
- LOG.info("Attempting to preempt for {}, pendingPreemptions={} on hosts={}",
- taskInfo.task, pendingPreemptions.get(), Arrays.toString(potentialHosts));
- preemptTasks(entry.getKey().getPriority(), 1, potentialHosts);
- }
- } else {
- // Request for a preemption if there's none pending. If a single preemption is pending,
- // and this is the next task to be assigned, it will be assigned once that slot becomes available.
- if (pendingPreemptions.get() == 0) {
- LOG.info("Attempting to preempt for {}, pendingPreemptions={} on any host",
- taskInfo.task, pendingPreemptions.get());
- preemptTasks(entry.getKey().getPriority(), 1, null);
- }
- }
- // Since there was an allocation failure - don't try assigning tasks at the next priority.
-
- scheduledAllAtPriority = false;
- // Don't break if this allocation failure was a result of a LOCALITY_DELAY. Others could still be allocated.
- if (scheduleResult != ScheduleResult.DELAYED_LOCALITY) {
- break;
- }
- } // end of else - i.e. could not allocate
- } // end of loop over pending tasks
- if (taskListAtPriority.isEmpty()) {
- // Remove the entry, if there's nothing left at the specific priority level
- pendingIterator.remove();
- }
- if (!scheduledAllAtPriority) {
- // Don't attempt scheduling for additional priorities
- break;
- }
- }
- } finally {
- writeLock.unlock();
- }
- }
-
-
- private ScheduleResult scheduleTask(TaskInfo taskInfo) {
- SelectHostResult selectHostResult = selectHost(taskInfo);
- if (selectHostResult.scheduleResult == ScheduleResult.SCHEDULED) {
- NodeServiceInstancePair nsPair = selectHostResult.nodeServiceInstancePair;
- Container container =
- containerFactory.createContainer(resourcePerExecutor, taskInfo.priority,
- nsPair.getServiceInstance().getHost(),
- nsPair.getServiceInstance().getRpcPort());
- writeLock.lock(); // While updating local structures
- try {
- LOG.info("Assigned task {} to container {}", taskInfo, container.getId());
- dagStats.registerTaskAllocated(taskInfo.requestedHosts, taskInfo.requestedRacks,
- nsPair.getServiceInstance().getHost());
- taskInfo.setAssignmentInfo(nsPair.getServiceInstance(), container.getId(), clock.getTime());
- registerRunningTask(taskInfo);
- nsPair.getNodeInfo().registerTaskScheduled();
- } finally {
- writeLock.unlock();
- }
- getContext().taskAllocated(taskInfo.task, taskInfo.clientCookie, container);
- }
- return selectHostResult.scheduleResult;
- }
-
- // Removes tasks from the runningList and sends out a preempt request to the system.
- // Subsequent tasks will be scheduled again once the de-allocate request for the preempted
- // task is processed.
- private void preemptTasks(int forPriority, int numTasksToPreempt, String []potentialHosts) {
- Set<String> preemptHosts;
- if (potentialHosts == null) {
- preemptHosts = null;
- } else {
- preemptHosts = Sets.newHashSet(potentialHosts);
- }
- writeLock.lock();
- List<TaskInfo> preemptedTaskList = null;
- try {
- NavigableMap<Integer, TreeSet<TaskInfo>> orderedMap = runningTasks.descendingMap();
- Iterator<Entry<Integer, TreeSet<TaskInfo>>> iterator = orderedMap.entrySet().iterator();
- int preemptedCount = 0;
- while (iterator.hasNext() && preemptedCount < numTasksToPreempt) {
- Entry<Integer, TreeSet<TaskInfo>> entryAtPriority = iterator.next();
- if (entryAtPriority.getKey() > forPriority) {
- Iterator<TaskInfo> taskInfoIterator = entryAtPriority.getValue().iterator();
- while (taskInfoIterator.hasNext() && preemptedCount < numTasksToPreempt) {
- TaskInfo taskInfo = taskInfoIterator.next();
- if (preemptHosts == null || preemptHosts.contains(taskInfo.assignedInstance.getHost())) {
- // Candidate for preemption.
- preemptedCount++;
- LOG.info("preempting {} for task at priority {} with potentialHosts={}", taskInfo,
- forPriority, potentialHosts == null ? "" : Arrays.toString(potentialHosts));
- taskInfo.setPreemptedInfo(clock.getTime());
- if (preemptedTaskList == null) {
- preemptedTaskList = new LinkedList<>();
- }
- dagStats.registerTaskPreempted(taskInfo.assignedInstance.getHost());
- preemptedTaskList.add(taskInfo);
- registerPendingPreemption(taskInfo.assignedInstance.getHost());
- // Remove from the runningTaskList
- taskInfoIterator.remove();
- }
- }
-
- // Remove entire priority level if it's been emptied.
- if (entryAtPriority.getValue().isEmpty()) {
- iterator.remove();
- }
- } else {
- // No tasks qualify as preemptable
- LOG.info("DBG: No tasks qualify as killable to schedule tasks at priority {}", forPriority);
- break;
- }
- }
- } finally {
- writeLock.unlock();
- }
- // Send out the preempted request outside of the lock.
- if (preemptedTaskList != null) {
- for (TaskInfo taskInfo : preemptedTaskList) {
- LOG.info("DBG: Preempting task {}", taskInfo);
- getContext().preemptContainer(taskInfo.containerId);
- // Preemption will finally be registered as a deallocateTask as a result of preemptContainer
- // That resets preemption info and allows additional tasks to be pre-empted if required.
- }
- }
- // The schedule loop will be triggered again when the deallocateTask request comes in for the
- // preempted task.
- }
-
- private void registerPendingPreemption(String host) {
- writeLock.lock();
- try {
- pendingPreemptions.incrementAndGet();
- MutableInt val = pendingPreemptionsPerHost.get(host);
- if (val == null) {
- val = new MutableInt(1);
- pendingPreemptionsPerHost.put(host, val);
- }
- val.increment();
- } finally {
- writeLock.unlock();
- }
- }
-
- private void unregisterPendingPreemption(String host) {
- writeLock.lock();
- try {
- pendingPreemptions.decrementAndGet();
- MutableInt val = pendingPreemptionsPerHost.get(host);
- Preconditions.checkNotNull(val);
- val.decrement();
- // Not bothering with removing the entry. There's a limited number of hosts, and a good
- // chance that the entry will make it back in when the AM is used for a long duration.
- } finally {
- writeLock.unlock();
- }
- }
-
- private class NodeEnablerCallable implements Callable<Void> {
-
- private AtomicBoolean isShutdown = new AtomicBoolean(false);
- private static final long REFRESH_INTERVAL = 10000l;
- long nextPollInterval = REFRESH_INTERVAL;
- long lastRefreshTime;
-
- @Override
- public Void call() {
-
- lastRefreshTime = System.currentTimeMillis();
- while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
- try {
- while (true) {
- NodeInfo nodeInfo = disabledNodesQueue.poll(nextPollInterval, TimeUnit.MILLISECONDS);
- if (nodeInfo != null) {
- long currentTime = System.currentTimeMillis();
- // A node became available. Enable the node and try scheduling.
- reenableDisabledNode(nodeInfo);
- trySchedulingPendingTasks();
-
- nextPollInterval -= (currentTime - lastRefreshTime);
- }
-
- if (nextPollInterval < 0 || nodeInfo == null) {
- // timeout expired. Reset the poll interval and refresh nodes.
- nextPollInterval = REFRESH_INTERVAL;
- lastRefreshTime = System.currentTimeMillis();
- // TODO Get rid of this polling once we have notificaitons from the registry sub-system
- if (LOG.isDebugEnabled()) {
- LOG.debug("Refreshing instances based on poll interval");
- }
- refreshInstances();
- scanForNodeChanges();
- }
- }
- } catch (InterruptedException e) {
- if (isShutdown.get()) {
- LOG.info("NodeEnabler thread interrupted after shutdown");
- break;
- } else {
- LOG.warn("NodeEnabler thread interrupted without being shutdown");
- throw new RuntimeException("NodeEnabler thread interrupted without being shutdown", e);
- }
- }
- }
- return null;
- }
-
- // Call this first, then send in an interrupt to the thread.
- public void shutdown() {
- isShutdown.set(true);
- }
- }
-
- private void trySchedulingPendingTasks() {
- scheduleLock.lock();
- try {
- pendingScheduleInvodations.set(true);
- scheduleCondition.signal();
- } finally {
- scheduleLock.unlock();
- }
- }
-
- private class SchedulerCallable implements Callable<Void> {
- private AtomicBoolean isShutdown = new AtomicBoolean(false);
-
- @Override
- public Void call() {
- while (!isShutdown.get() && !Thread.currentThread().isInterrupted()) {
- scheduleLock.lock();
- try {
- while (!pendingScheduleInvodations.get()) {
- scheduleCondition.await();
- }
- } catch (InterruptedException e) {
- if (isShutdown.get()) {
- LOG.info("Scheduler thread interrupted after shutdown");
- break;
- } else {
- LOG.warn("Scheduler thread interrupted without being shutdown");
- throw new RuntimeException("Scheduler thread interrupted without being shutdown", e);
- }
- } finally {
- scheduleLock.unlock();
- }
-
- // Set pending to false since scheduling is about to run. Any triggers up to this point
- // will be handled in the next run.
- // A new request may come in right after this is set to false, but before the actual scheduling.
- // This will be handled in this run, but will cause an immediate run after, which is harmless.
- pendingScheduleInvodations.set(false);
- // Schedule outside of the scheduleLock - which should only be used to wait on the condition.
- schedulePendingTasks();
- }
- return null;
- }
-
- // Call this first, then send in an interrupt to the thread.
- public void shutdown() {
- isShutdown.set(true);
- }
- }
-
- @VisibleForTesting
- static class NodeInfo implements Delayed {
- private final NodeBlacklistConf blacklistConf;
- final ServiceInstance serviceInstance;
- private final Clock clock;
-
- long expireTimeMillis = -1;
- private long numSuccessfulTasks = 0;
- private long numSuccessfulTasksAtLastBlacklist = -1;
- float cumulativeBackoffFactor = 1.0f;
-
- // Indicates whether a node had a recent communication failure.
- private boolean hadCommFailure = false;
-
- // Indicates whether a node is disabled - for whatever reason - commFailure, busy, etc.
- private boolean disabled = false;
-
- private int numPreemptedTasks = 0;
- private int numScheduledTasks = 0;
- private final int numSchedulableTasks;
-
-
- /**
- * Create a NodeInfo bound to a service instance
- *
- * @param serviceInstance the associated serviceInstance
- * @param blacklistConf blacklist configuration
- * @param clock clock to use to obtain timing information
- * @param numSchedulableTasksConf number of schedulable tasks on the node. 0 represents auto
- * detect based on the serviceInstance, -1 indicates indicates
- * unlimited capacity
- */
- NodeInfo(ServiceInstance serviceInstance, NodeBlacklistConf blacklistConf, Clock clock, int numSchedulableTasksConf) {
- Preconditions.checkArgument(numSchedulableTasksConf >= -1, "NumSchedulableTasks must be >=-1");
- this.serviceInstance = serviceInstance;
- this.blacklistConf = blacklistConf;
- this.clock = clock;
-
- if (numSchedulableTasksConf == 0) {
- int pendingQueueuCapacity = 0;
- String pendingQueueCapacityString = serviceInstance.getProperties()
- .get(ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE.varname);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting up node: " + serviceInstance + ", with available capacity=" +
- serviceInstance.getResource().getVirtualCores() + ", pendingQueueCapacity=" +
- pendingQueueCapacityString);
- }
-
- if (pendingQueueCapacityString != null) {
- pendingQueueuCapacity = Integer.parseInt(pendingQueueCapacityString);
- }
- this.numSchedulableTasks = serviceInstance.getResource().getVirtualCores() + pendingQueueuCapacity;
- } else {
- this.numSchedulableTasks = numSchedulableTasksConf;
- }
- LOG.info("Setting up node: " + serviceInstance + " with schedulableCapacity=" + this.numSchedulableTasks);
- }
-
- ServiceInstance getServiceInstance() {
- return serviceInstance;
- }
-
- void enableNode() {
- expireTimeMillis = -1;
- disabled = false;
- hadCommFailure = false;
- }
-
- void disableNode(boolean commFailure) {
- long duration = blacklistConf.minDelay;
- long currentTime = clock.getTime();
- this.hadCommFailure = commFailure;
- disabled = true;
- if (numSuccessfulTasksAtLastBlacklist == numSuccessfulTasks) {
- // Relying on a task succeeding to reset the exponent.
- // There's no notifications on whether a task gets accepted or not. That would be ideal to
- // reset this.
- cumulativeBackoffFactor = cumulativeBackoffFactor * blacklistConf.backoffFactor;
- } else {
- // Was able to execute something before the last blacklist. Reset the exponent.
- cumulativeBackoffFactor = 1.0f;
- }
-
- long delayTime = (long) (duration * cumulativeBackoffFactor);
- if (delayTime > blacklistConf.maxDelay) {
- delayTime = blacklistConf.maxDelay;
- }
- if (LOG.isInfoEnabled()) {
- LOG.info("Disabling instance " + serviceInstance + " for " + delayTime + " milli-seconds");
- }
- expireTimeMillis = currentTime + delayTime;
- numSuccessfulTasksAtLastBlacklist = numSuccessfulTasks;
- }
-
- void registerTaskScheduled() {
- numScheduledTasks++;
- }
-
- void registerTaskSuccess() {
- numSuccessfulTasks++;
- numScheduledTasks--;
- }
-
- void registerUnsuccessfulTaskEnd(boolean wasPreempted) {
- numScheduledTasks--;
- if (wasPreempted) {
- numPreemptedTasks++;
- }
- }
-
- public boolean isDisabled() {
- return disabled;
- }
-
- public boolean hadCommFailure() {
- return hadCommFailure;
- }
- /* Returning true does not guarantee that the task will run, considering other queries
- may be running in the system. Also depends upon the capacity usage configuration
- */
- public boolean canAcceptTask() {
- boolean result = !hadCommFailure && !disabled && serviceInstance.isAlive()
- &&(numSchedulableTasks == -1 || ((numSchedulableTasks - numScheduledTasks) > 0));
- LOG.info("canAcceptTask={}, numScheduledTasks={}, numSchedulableTasks={}, hadCommFailure={}, disabled={}, serviceInstance.isAlive={}", result, numScheduledTasks, numSchedulableTasks, hadCommFailure, disabled, serviceInstance.isAlive());
- return result;
- }
-
- @Override
- public long getDelay(TimeUnit unit) {
- return unit.convert(expireTimeMillis - clock.getTime(), TimeUnit.MILLISECONDS);
- }
-
- @Override
- public int compareTo(Delayed o) {
- NodeInfo other = (NodeInfo) o;
- if (other.expireTimeMillis > this.expireTimeMillis) {
- return -1;
- } else if (other.expireTimeMillis < this.expireTimeMillis) {
- return 1;
- } else {
- return 0;
- }
- }
-
- @Override
- public String toString() {
- return "NodeInfo{" + "instance=" + serviceInstance
- + ", expireTimeMillis=" + expireTimeMillis + ", numSuccessfulTasks=" + numSuccessfulTasks
- + ", numSuccessfulTasksAtLastBlacklist=" + numSuccessfulTasksAtLastBlacklist
- + ", cumulativeBackoffFactor=" + cumulativeBackoffFactor
- + ", numSchedulableTasks=" + numSchedulableTasks
- + ", numScheduledTasks=" + numScheduledTasks
- + ", disabled=" + disabled
- + ", commFailures=" + hadCommFailure
- +'}';
- }
- }
-
- @VisibleForTesting
- static class StatsPerDag {
- int numRequestedAllocations = 0;
- int numRequestsWithLocation = 0;
- int numRequestsWithoutLocation = 0;
- int numTotalAllocations = 0;
- int numLocalAllocations = 0;
- int numNonLocalAllocations = 0;
- int numAllocationsNoLocalityRequest = 0;
- int numRejectedTasks = 0;
- int numCommFailures = 0;
- int numDelayedAllocations = 0;
- int numPreemptedTasks = 0;
- Map<String, AtomicInteger> localityBasedNumAllocationsPerHost = new HashMap<>();
- Map<String, AtomicInteger> numAllocationsPerHost = new HashMap<>();
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("NumPreemptedTasks=").append(numPreemptedTasks).append(", ");
- sb.append("NumRequestedAllocations=").append(numRequestedAllocations).append(", ");
- sb.append("NumRequestsWithlocation=").append(numRequestsWithLocation).append(", ");
- sb.append("NumLocalAllocations=").append(numLocalAllocations).append(",");
- sb.append("NumNonLocalAllocations=").append(numNonLocalAllocations).append(",");
- sb.append("NumTotalAllocations=").append(numTotalAllocations).append(",");
- sb.append("NumRequestsWithoutLocation=").append(numRequestsWithoutLocation).append(", ");
- sb.append("NumRejectedTasks=").append(numRejectedTasks).append(", ");
- sb.append("NumCommFailures=").append(numCommFailures).append(", ");
- sb.append("NumDelayedAllocations=").append(numDelayedAllocations).append(", ");
- sb.append("LocalityBasedAllocationsPerHost=").append(localityBasedNumAllocationsPerHost)
- .append(", ");
- sb.append("NumAllocationsPerHost=").append(numAllocationsPerHost);
- return sb.toString();
- }
-
- void registerTaskRequest(String[] requestedHosts, String[] requestedRacks) {
- numRequestedAllocations++;
- // TODO Change after HIVE-9987. For now, there's no rack matching.
- if (requestedHosts != null && requestedHosts.length != 0) {
- numRequestsWithLocation++;
- } else {
- numRequestsWithoutLocation++;
- }
- }
-
- void registerTaskAllocated(String[] requestedHosts, String[] requestedRacks,
- String allocatedHost) {
- // TODO Change after HIVE-9987. For now, there's no rack matching.
- if (requestedHosts != null && requestedHosts.length != 0) {
- Set<String> requestedHostSet = new HashSet<>(Arrays.asList(requestedHosts));
- if (requestedHostSet.contains(allocatedHost)) {
- numLocalAllocations++;
- _registerAllocationInHostMap(allocatedHost, localityBasedNumAllocationsPerHost);
- } else {
- numNonLocalAllocations++;
- }
- } else {
- numAllocationsNoLocalityRequest++;
- }
- numTotalAllocations++;
- _registerAllocationInHostMap(allocatedHost, numAllocationsPerHost);
- }
-
- void registerTaskPreempted(String host) {
- numPreemptedTasks++;
- }
-
- void registerCommFailure(String host) {
- numCommFailures++;
- }
-
- void registerTaskRejected(String host) {
- numRejectedTasks++;
- }
-
- void registerDelayedAllocation() {
- numDelayedAllocations++;
- }
-
- private void _registerAllocationInHostMap(String host, Map<String, AtomicInteger> hostMap) {
- AtomicInteger val = hostMap.get(host);
- if (val == null) {
- val = new AtomicInteger(0);
- hostMap.put(host, val);
- }
- val.incrementAndGet();
- }
- }
-
- private static class TaskInfo {
- // IDs used to ensure two TaskInfos are different without using the underlying task instance.
- // Required for insertion into a TreeMap
- static final AtomicLong ID_GEN = new AtomicLong(0);
- final long uniqueId;
- final Object task;
- final Object clientCookie;
- final Priority priority;
- final Resource capability;
- final String[] requestedHosts;
- final String[] requestedRacks;
- final long requestTime;
- long startTime;
- long preemptTime;
- ContainerId containerId;
- ServiceInstance assignedInstance;
- private boolean assigned = false;
- private boolean preempted = false;
-
- private int numAssignAttempts = 0;
-
- // TaskInfo instances for two different tasks will not be the same. Only a single instance should
- // ever be created for a taskAttempt
- public TaskInfo(Object task, Object clientCookie, Priority priority, Resource capability,
- String[] hosts, String[] racks, long requestTime) {
- this.task = task;
- this.clientCookie = clientCookie;
- this.priority = priority;
- this.capability = capability;
- this.requestedHosts = hosts;
- this.requestedRacks = racks;
- this.requestTime = requestTime;
- this.uniqueId = ID_GEN.getAndIncrement();
- }
-
- void setAssignmentInfo(ServiceInstance instance, ContainerId containerId, long startTime) {
- this.assignedInstance = instance;
- this.containerId = containerId;
- this.startTime = startTime;
- assigned = true;
- }
-
- void setPreemptedInfo(long preemptTime) {
- this.preempted = true;
- this.assigned = false;
- this.preemptTime = preemptTime;
- }
-
- void triedAssigningTask() {
- numAssignAttempts++;
- }
-
- int getNumPreviousAssignAttempts() {
- return numAssignAttempts;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- TaskInfo taskInfo = (TaskInfo) o;
-
- if (uniqueId != taskInfo.uniqueId) {
- return false;
- }
- return task.equals(taskInfo.task);
-
- }
-
- @Override
- public int hashCode() {
- int result = (int) (uniqueId ^ (uniqueId >>> 32));
- result = 31 * result + task.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "TaskInfo{" +
- "task=" + task +
- ", priority=" + priority +
- ", startTime=" + startTime +
- ", containerId=" + containerId +
- ", assignedInstance=" + assignedInstance +
- ", uniqueId=" + uniqueId +
- '}';
- }
- }
-
- // Newer tasks first.
- private static class TaskStartComparator implements Comparator<TaskInfo> {
-
- @Override
- public int compare(TaskInfo o1, TaskInfo o2) {
- if (o1.startTime > o2.startTime) {
- return -1;
- } else if (o1.startTime < o2.startTime) {
- return 1;
- } else {
- // Comparing on time is not sufficient since two may be created at the same time,
- // in which case inserting into a TreeSet/Map would break
- if (o1.uniqueId > o2.uniqueId) {
- return -1;
- } else if (o1.uniqueId < o2.uniqueId) {
- return 1;
- } else {
- return 0;
- }
- }
- }
- }
-
- private static class SelectHostResult {
- final NodeServiceInstancePair nodeServiceInstancePair;
- final ScheduleResult scheduleResult;
-
- SelectHostResult(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
- this.nodeServiceInstancePair = new NodeServiceInstancePair(serviceInstance, nodeInfo);
- this.scheduleResult = ScheduleResult.SCHEDULED;
- }
-
- SelectHostResult(ScheduleResult scheduleResult) {
- this.nodeServiceInstancePair = null;
- this.scheduleResult = scheduleResult;
- }
- }
-
- private static final SelectHostResult SELECT_HOST_RESULT_INADEQUATE_TOTAL_CAPACITY =
- new SelectHostResult(ScheduleResult.INADEQUATE_TOTAL_RESOURCES);
- private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_LOCALITY =
- new SelectHostResult(ScheduleResult.DELAYED_LOCALITY);
- private static final SelectHostResult SELECT_HOST_RESULT_DELAYED_RESOURCES =
- new SelectHostResult(ScheduleResult.DELAYED_RESOURCES);
-
- private static class NodeServiceInstancePair {
- final NodeInfo nodeInfo;
- final ServiceInstance serviceInstance;
-
- private NodeServiceInstancePair(ServiceInstance serviceInstance, NodeInfo nodeInfo) {
- this.serviceInstance = serviceInstance;
- this.nodeInfo = nodeInfo;
- }
-
- public ServiceInstance getServiceInstance() {
- return serviceInstance;
- }
-
- public NodeInfo getNodeInfo() {
- return nodeInfo;
- }
- }
-
- private static final class NodeBlacklistConf {
- private final long minDelay;
- private final long maxDelay;
- private final float backoffFactor;
-
- public NodeBlacklistConf(long minDelay, long maxDelay, float backoffFactor) {
- this.minDelay = minDelay;
- this.maxDelay = maxDelay;
- this.backoffFactor = backoffFactor;
- }
-
- @Override
- public String toString() {
- return "NodeBlacklistConf{" +
- "minDelay=" + minDelay +
- ", maxDelay=" + maxDelay +
- ", backoffFactor=" + backoffFactor +
- '}';
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/protobuf/LlapDaemonProtocol.proto
----------------------------------------------------------------------
diff --git a/llap-server/src/protobuf/LlapDaemonProtocol.proto b/llap-server/src/protobuf/LlapDaemonProtocol.proto
deleted file mode 100644
index 944c96c..0000000
--- a/llap-server/src/protobuf/LlapDaemonProtocol.proto
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.hive.llap.daemon.rpc";
-option java_outer_classname = "LlapDaemonProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-
-// TODO Change this as the interface evolves. Currently using Tez constructs.
-
-message UserPayloadProto {
- optional bytes user_payload = 1;
- optional int32 version = 2;
-}
-
-message EntityDescriptorProto {
- optional string class_name = 1;
- optional UserPayloadProto user_payload = 2;
- optional bytes history_text = 3;
-}
-
-message IOSpecProto {
- optional string connected_vertex_name = 1;
- optional EntityDescriptorProto io_descriptor = 2;
- optional int32 physical_edge_count = 3;
-}
-
-message GroupInputSpecProto {
- optional string group_name = 1;
- repeated string group_vertices = 2;
- optional EntityDescriptorProto merged_input_descriptor = 3;
-}
-
-
-message FragmentSpecProto {
- optional string fragment_identifier_string = 1;
- optional string dag_name = 2;
- optional int32 dag_id = 11;
- optional string vertex_name = 3;
- optional EntityDescriptorProto processor_descriptor = 4;
- repeated IOSpecProto input_specs = 5;
- repeated IOSpecProto output_specs = 6;
- repeated GroupInputSpecProto grouped_input_specs = 7;
- optional int32 vertex_parallelism = 8;
- optional int32 fragment_number =9;
- optional int32 attempt_number = 10;
-}
-
-message FragmentRuntimeInfo {
- optional int32 num_self_and_upstream_tasks = 1;
- optional int32 num_self_and_upstream_completed_tasks = 2;
- optional int32 within_dag_priority = 3;
- optional int64 dag_start_time = 4;
- optional int64 first_attempt_start_time = 5;
- optional int64 current_attempt_start_time = 6;
-}
-
-enum SourceStateProto {
- S_SUCCEEDED = 1;
- S_RUNNING = 2;
-}
-
-message QueryIdentifierProto {
- optional string app_identifier = 1;
- optional int32 dag_identifier = 2;
-}
-
-message SubmitWorkRequestProto {
- optional string container_id_string = 1;
- optional string am_host = 2;
- optional int32 am_port = 3;
- optional string token_identifier = 4;
- optional bytes credentials_binary = 5;
- optional string user = 6;
- optional string application_id_string = 7;
- optional int32 app_attempt_number = 8;
- optional FragmentSpecProto fragment_spec = 9;
- optional FragmentRuntimeInfo fragment_runtime_info = 10;
-}
-
-enum SubmissionStateProto {
- ACCEPTED = 1;
- REJECTED = 2;
- EVICTED_OTHER = 3;
-}
-
-message SubmitWorkResponseProto {
- optional SubmissionStateProto submission_state = 1;
-}
-
-message SourceStateUpdatedRequestProto {
- optional QueryIdentifierProto query_identifier = 1;
- optional string src_name = 2;
- optional SourceStateProto state = 3;
-}
-
-message SourceStateUpdatedResponseProto {
-}
-
-message QueryCompleteRequestProto {
- optional string query_id = 1;
- optional QueryIdentifierProto query_identifier = 2;
- optional int64 delete_delay = 4 [default = 0];
-}
-
-message QueryCompleteResponseProto {
-}
-
-message TerminateFragmentRequestProto {
- optional QueryIdentifierProto query_identifier = 1;
- optional string fragment_identifier_string = 2;
-}
-
-message TerminateFragmentResponseProto {
-}
-
-message GetTokenRequestProto {
-}
-
-message GetTokenResponseProto {
- optional bytes token = 1;
-}
-
-service LlapDaemonProtocol {
- rpc submitWork(SubmitWorkRequestProto) returns (SubmitWorkResponseProto);
- rpc sourceStateUpdated(SourceStateUpdatedRequestProto) returns (SourceStateUpdatedResponseProto);
- rpc queryComplete(QueryCompleteRequestProto) returns (QueryCompleteResponseProto);
- rpc terminateFragment(TerminateFragmentRequestProto) returns (TerminateFragmentResponseProto);
-}
-
-service LlapManagementProtocol {
- rpc getDelegationToken(GetTokenRequestProto) returns (GetTokenResponseProto);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
index 44c958d..1cef218 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
@@ -27,10 +27,11 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
-import org.apache.hadoop.hive.llap.daemon.LlapDaemonProtocolBlockingPB;
+import org.apache.hadoop.hive.llap.protocol.LlapProtocolBlockingPB;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmissionStateProto;
+import org.apache.hadoop.hive.llap.impl.LlapProtocolClientImpl;
import org.junit.Test;
public class TestLlapDaemonProtocolServerImpl {
@@ -42,8 +43,8 @@ public class TestLlapDaemonProtocolServerImpl {
int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT);
int numHandlers = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_NUM_HANDLERS);
ContainerRunner containerRunnerMock = mock(ContainerRunner.class);
- LlapDaemonProtocolServerImpl server =
- new LlapDaemonProtocolServerImpl(numHandlers, containerRunnerMock,
+ LlapProtocolServerImpl server =
+ new LlapProtocolServerImpl(numHandlers, containerRunnerMock,
new AtomicReference<InetSocketAddress>(), new AtomicReference<InetSocketAddress>(),
rpcPort, rpcPort + 1);
when(containerRunnerMock.submitWork(any(SubmitWorkRequestProto.class))).thenReturn(
@@ -56,8 +57,8 @@ public class TestLlapDaemonProtocolServerImpl {
server.start();
InetSocketAddress serverAddr = server.getBindAddress();
- LlapDaemonProtocolBlockingPB client =
- new LlapDaemonProtocolClientImpl(new Configuration(), serverAddr.getHostName(),
+ LlapProtocolBlockingPB client =
+ new LlapProtocolClientImpl(new Configuration(), serverAddr.getHostName(),
serverAddr.getPort(), null, null);
SubmitWorkResponseProto responseProto = client.submitWork(null,
SubmitWorkRequestProto.newBuilder()
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java
deleted file mode 100644
index a6af8c2..0000000
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapDaemonProtocolClientProxy.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.protobuf.Message;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.junit.Test;
-
-public class TestLlapDaemonProtocolClientProxy {
-
- @Test (timeout = 5000)
- public void testMultipleNodes() {
- RequestManagerForTest requestManager = new RequestManagerForTest(1);
-
- LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
- LlapNodeId nodeId2 = LlapNodeId.getInstance("host2", 1025);
-
- Message mockMessage = mock(Message.class);
- LlapDaemonProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
- LlapDaemonProtocolClientProxy.ExecuteRequestCallback.class);
-
- // Request two messages
- requestManager.queueRequest(
- new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
- requestManager.queueRequest(
- new CallableRequestForTest(nodeId2, mockMessage, mockExecuteRequestCallback));
-
- // Should go through in a single process call
- requestManager.process();
- assertEquals(2, requestManager.numSubmissionsCounters);
- assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
- assertNotNull(requestManager.numInvocationsPerNode.get(nodeId2));
- assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
- assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId2).getValue().intValue());
- assertEquals(0, requestManager.currentLoopSkippedRequests.size());
- assertEquals(0, requestManager.currentLoopSkippedRequests.size());
- assertEquals(0, requestManager.currentLoopDisabledNodes.size());
- }
-
- @Test(timeout = 5000)
- public void testSingleInvocationPerNode() {
- RequestManagerForTest requestManager = new RequestManagerForTest(1);
-
- LlapNodeId nodeId1 = LlapNodeId.getInstance("host1", 1025);
-
- Message mockMessage = mock(Message.class);
- LlapDaemonProtocolClientProxy.ExecuteRequestCallback mockExecuteRequestCallback = mock(
- LlapDaemonProtocolClientProxy.ExecuteRequestCallback.class);
-
- // First request for host.
- requestManager.queueRequest(
- new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
- requestManager.process();
- assertEquals(1, requestManager.numSubmissionsCounters);
- assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
- assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
- assertEquals(0, requestManager.currentLoopSkippedRequests.size());
-
- // Second request for host. Single invocation since the last has not completed.
- requestManager.queueRequest(
- new CallableRequestForTest(nodeId1, mockMessage, mockExecuteRequestCallback));
- requestManager.process();
- assertEquals(1, requestManager.numSubmissionsCounters);
- assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
- assertEquals(1, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
- assertEquals(1, requestManager.currentLoopSkippedRequests.size());
- assertEquals(1, requestManager.currentLoopDisabledNodes.size());
- assertTrue(requestManager.currentLoopDisabledNodes.contains(nodeId1));
-
- // Complete first request. Second pending request should go through.
- requestManager.requestFinished(nodeId1);
- requestManager.process();
- assertEquals(2, requestManager.numSubmissionsCounters);
- assertNotNull(requestManager.numInvocationsPerNode.get(nodeId1));
- assertEquals(2, requestManager.numInvocationsPerNode.get(nodeId1).getValue().intValue());
- assertEquals(0, requestManager.currentLoopSkippedRequests.size());
- assertEquals(0, requestManager.currentLoopDisabledNodes.size());
- assertFalse(requestManager.currentLoopDisabledNodes.contains(nodeId1));
- }
-
-
- static class RequestManagerForTest extends LlapDaemonProtocolClientProxy.RequestManager {
-
- int numSubmissionsCounters = 0;
- private Map<LlapNodeId, MutableInt> numInvocationsPerNode = new HashMap<>();
-
- public RequestManagerForTest(int numThreads) {
- super(numThreads);
- }
-
- protected void submitToExecutor(LlapDaemonProtocolClientProxy.CallableRequest request, LlapNodeId nodeId) {
- numSubmissionsCounters++;
- MutableInt nodeCount = numInvocationsPerNode.get(nodeId);
- if (nodeCount == null) {
- nodeCount = new MutableInt(0);
- numInvocationsPerNode.put(nodeId, nodeCount);
- }
- nodeCount.increment();
- }
-
- void reset() {
- numSubmissionsCounters = 0;
- numInvocationsPerNode.clear();
- }
-
- }
-
- static class CallableRequestForTest extends LlapDaemonProtocolClientProxy.CallableRequest<Message, Message> {
-
- protected CallableRequestForTest(LlapNodeId nodeId, Message message,
- LlapDaemonProtocolClientProxy.ExecuteRequestCallback<Message> callback) {
- super(nodeId, message, callback);
- }
-
- @Override
- public Message call() throws Exception {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
deleted file mode 100644
index 8f3d104..0000000
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/tezplugins/TestLlapTaskCommunicator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.junit.Test;
-
-public class TestLlapTaskCommunicator {
-
- @Test (timeout = 5000)
- public void testEntityTracker1() {
- LlapTaskCommunicator.EntityTracker entityTracker = new LlapTaskCommunicator.EntityTracker();
-
- String host1 = "host1";
- String host2 = "host2";
- String host3 = "host3";
- int port = 1451;
-
-
- // Simple container registration and un-registration without any task attempt being involved.
- ContainerId containerId101 = constructContainerId(101);
- entityTracker.registerContainer(containerId101, host1, port);
- assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId101));
-
- entityTracker.unregisterContainer(containerId101);
- assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
- assertNull(entityTracker.getNodeIdForContainer(containerId101));
- assertEquals(0, entityTracker.nodeMap.size());
- assertEquals(0, entityTracker.attemptToNodeMap.size());
- assertEquals(0, entityTracker.containerToNodeMap.size());
-
-
- // Simple task registration and un-registration.
- ContainerId containerId1 = constructContainerId(1);
- TezTaskAttemptID taskAttemptId1 = constructTaskAttemptId(1);
- entityTracker.registerTaskAttempt(containerId1, taskAttemptId1, host1, port);
- assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId1));
- assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
-
- entityTracker.unregisterTaskAttempt(taskAttemptId1);
- assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
- assertNull(entityTracker.getNodeIdForContainer(containerId1));
- assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId1));
- assertEquals(0, entityTracker.nodeMap.size());
- assertEquals(0, entityTracker.attemptToNodeMap.size());
- assertEquals(0, entityTracker.containerToNodeMap.size());
-
- // Register taskAttempt, unregister container. TaskAttempt should also be unregistered
- ContainerId containerId201 = constructContainerId(201);
- TezTaskAttemptID taskAttemptId201 = constructTaskAttemptId(201);
- entityTracker.registerTaskAttempt(containerId201, taskAttemptId201, host1, port);
- assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForContainer(containerId201));
- assertEquals(LlapNodeId.getInstance(host1, port), entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
-
- entityTracker.unregisterContainer(containerId201);
- assertNull(entityTracker.getContainerAttemptMapForNode(LlapNodeId.getInstance(host1, port)));
- assertNull(entityTracker.getNodeIdForContainer(containerId201));
- assertNull(entityTracker.getNodeIdForTaskAttempt(taskAttemptId201));
- assertEquals(0, entityTracker.nodeMap.size());
- assertEquals(0, entityTracker.attemptToNodeMap.size());
- assertEquals(0, entityTracker.containerToNodeMap.size());
-
- entityTracker.unregisterTaskAttempt(taskAttemptId201); // No errors
- }
-
-
- private ContainerId constructContainerId(int id) {
- ContainerId containerId = mock(ContainerId.class);
- doReturn(id).when(containerId).getId();
- doReturn((long)id).when(containerId).getContainerId();
- return containerId;
- }
-
- private TezTaskAttemptID constructTaskAttemptId(int id) {
- TezTaskAttemptID taskAttemptId = mock(TezTaskAttemptID.class);
- doReturn(id).when(taskAttemptId).getId();
- return taskAttemptId;
- }
-
-}
[11/11] hive git commit: HIVE-12934 branch-2.0 addendum. Update
version fro new modules to reflect branch version. (Siddharth Seth)
Posted by ss...@apache.org.
HIVE-12934 branch-2.0 addendum. Update version fro new modules to
reflect branch version. (Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/98d51677
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/98d51677
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/98d51677
Branch: refs/heads/branch-2.0
Commit: 98d516770756df44bbf13f09cfa1cf1d7e4508d2
Parents: 9886414
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 1 09:57:26 2016 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Feb 1 09:57:26 2016 -0800
----------------------------------------------------------------------
llap-common/pom.xml | 2 +-
llap-tez/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/98d51677/llap-common/pom.xml
----------------------------------------------------------------------
diff --git a/llap-common/pom.xml b/llap-common/pom.xml
index 5343479..ac675cd 100644
--- a/llap-common/pom.xml
+++ b/llap-common/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/98d51677/llap-tez/pom.xml
----------------------------------------------------------------------
diff --git a/llap-tez/pom.xml b/llap-tez/pom.xml
index ce020da..812d429 100644
--- a/llap-tez/pom.xml
+++ b/llap-tez/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
[05/11] hive git commit: HIVE-12934. Refactor llap module structure
to allow for a usable client. (Siddharth Seth,
reviewed by Sergey Shelukhin) (cherry picked from commit
4185d9b8e2eecfef3b5a38899f6928fa82c01e99)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
deleted file mode 100644
index eb6384f..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++ /dev/null
@@ -1,755 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryCompleteRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentResponseProto;
-import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.hive.llap.tezplugins.helpers.SourceStateTracker;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.tez.common.TezTaskUmbilicalProtocol;
-import org.apache.tez.common.security.JobTokenSecretManager;
-import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.event.VertexStateUpdate;
-import org.apache.tez.dag.app.TezTaskCommunicatorImpl;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.runtime.api.impl.TaskSpec;
-import org.apache.tez.runtime.api.impl.TezHeartbeatRequest;
-import org.apache.tez.runtime.api.impl.TezHeartbeatResponse;
-import org.apache.tez.serviceplugins.api.ContainerEndReason;
-import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
-import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
-
- private static final Logger LOG = LoggerFactory.getLogger(LlapTaskCommunicator.class);
-
- private static final boolean isInfoEnabled = LOG.isInfoEnabled();
- private static final boolean isDebugEnabed = LOG.isDebugEnabled();
-
- private final SubmitWorkRequestProto BASE_SUBMIT_WORK_REQUEST;
-
- private final ConcurrentMap<QueryIdentifierProto, ByteBuffer> credentialMap;
-
- // Tracks containerIds and taskAttemptIds, so can be kept independent of the running DAG.
- // When DAG specific cleanup happens, it'll be better to link this to a DAG though.
- private final EntityTracker entityTracker = new EntityTracker();
- private final SourceStateTracker sourceStateTracker;
- private final Set<LlapNodeId> nodesForQuery = new HashSet<>();
-
- private LlapDaemonProtocolClientProxy communicator;
- private long deleteDelayOnDagComplete;
- private final LlapTaskUmbilicalProtocol umbilical;
- private final Token<LlapTokenIdentifier> token;
-
- // These two structures track the list of known nodes, and the list of nodes which are sending in keep-alive heartbeats.
- // Primarily for debugging purposes a.t.m, since there's some unexplained TASK_TIMEOUTS which are currently being observed.
- private final ConcurrentMap<LlapNodeId, Long> knownNodeMap = new ConcurrentHashMap<>();
- private final ConcurrentMap<LlapNodeId, PingingNodeInfo> pingedNodeMap = new ConcurrentHashMap<>();
-
-
- private volatile int currentDagId;
- private volatile QueryIdentifierProto currentQueryIdentifierProto;
-
- public LlapTaskCommunicator(
- TaskCommunicatorContext taskCommunicatorContext) {
- super(taskCommunicatorContext);
- Credentials credentials = taskCommunicatorContext.getCredentials();
- if (credentials != null) {
- @SuppressWarnings("unchecked")
- Token<LlapTokenIdentifier> llapToken =
- (Token<LlapTokenIdentifier>)credentials.getToken(LlapTokenIdentifier.KIND_NAME);
- this.token = llapToken;
- } else {
- this.token = null;
- }
- Preconditions.checkState((token != null) == UserGroupInformation.isSecurityEnabled());
-
- umbilical = new LlapTaskUmbilicalProtocolImpl(getUmbilical());
- SubmitWorkRequestProto.Builder baseBuilder = SubmitWorkRequestProto.newBuilder();
-
- // TODO Avoid reading this from the environment
- baseBuilder.setUser(System.getenv(ApplicationConstants.Environment.USER.name()));
- baseBuilder.setApplicationIdString(
- taskCommunicatorContext.getApplicationAttemptId().getApplicationId().toString());
- baseBuilder
- .setAppAttemptNumber(taskCommunicatorContext.getApplicationAttemptId().getAttemptId());
- baseBuilder.setTokenIdentifier(getTokenIdentifier());
-
- BASE_SUBMIT_WORK_REQUEST = baseBuilder.build();
-
- credentialMap = new ConcurrentHashMap<>();
- sourceStateTracker = new SourceStateTracker(getContext(), this);
- }
-
- @Override
- public void initialize() throws Exception {
- super.initialize();
- Configuration conf = getConf();
- int numThreads = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_COMMUNICATOR_NUM_THREADS);
- this.communicator = new LlapDaemonProtocolClientProxy(numThreads, conf, token);
- this.deleteDelayOnDagComplete = HiveConf.getTimeVar(
- conf, ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS);
- LOG.info("Running LlapTaskCommunicator with "
- + "fileCleanupDelay=" + deleteDelayOnDagComplete
- + ", numCommunicatorThreads=" + numThreads);
- this.communicator.init(conf);
- }
-
- @Override
- public void start() {
- super.start();
- this.communicator.start();
- }
-
- @Override
- public void shutdown() {
- super.shutdown();
- if (this.communicator != null) {
- this.communicator.stop();
- }
- }
-
- @Override
- protected void startRpcServer() {
- Configuration conf = getConf();
- try {
- JobTokenSecretManager jobTokenSecretManager =
- new JobTokenSecretManager();
- jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken);
-
- int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT,
- TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT);
- server = new RPC.Builder(conf)
- .setProtocol(LlapTaskUmbilicalProtocol.class)
- .setBindAddress("0.0.0.0")
- .setPort(0)
- .setInstance(umbilical)
- .setNumHandlers(numHandlers)
- .setSecretManager(jobTokenSecretManager).build();
-
- if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
- server.refreshServiceAcl(conf, new LlapUmbilicalPolicyProvider());
- }
-
- server.start();
- this.address = NetUtils.getConnectAddress(server);
- LOG.info(
- "Started LlapUmbilical: " + umbilical.getClass().getName() + " at address: " + address +
- " with numHandlers=" + numHandlers);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- }
-
- @Override
- public void registerRunningContainer(ContainerId containerId, String hostname, int port) {
- super.registerRunningContainer(containerId, hostname, port);
- entityTracker.registerContainer(containerId, hostname, port);
-
- }
-
- @Override
- public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, String diagnostics) {
- super.registerContainerEnd(containerId, endReason, diagnostics);
- if (endReason == ContainerEndReason.INTERNAL_PREEMPTION) {
- LOG.info("Processing containerEnd for container {} caused by internal preemption", containerId);
- TezTaskAttemptID taskAttemptId = entityTracker.getTaskAttemptIdForContainer(containerId);
- if (taskAttemptId != null) {
- sendTaskTerminated(taskAttemptId, true);
- }
- }
- entityTracker.unregisterContainer(containerId);
- }
-
-
- @Override
- public void registerRunningTaskAttempt(final ContainerId containerId, final TaskSpec taskSpec,
- Map<String, LocalResource> additionalResources,
- Credentials credentials,
- boolean credentialsChanged,
- int priority) {
- super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials,
- credentialsChanged, priority);
- int dagId = taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId();
- if (currentQueryIdentifierProto == null || (dagId != currentQueryIdentifierProto.getDagIdentifier())) {
- resetCurrentDag(dagId);
- }
-
-
- ContainerInfo containerInfo = getContainerInfo(containerId);
- String host;
- int port;
- if (containerInfo != null) {
- synchronized (containerInfo) {
- host = containerInfo.host;
- port = containerInfo.port;
- }
- } else {
- // TODO Handle this properly
- throw new RuntimeException("ContainerInfo not found for container: " + containerId +
- ", while trying to launch task: " + taskSpec.getTaskAttemptID());
- }
-
- LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
- registerKnownNode(nodeId);
- entityTracker.registerTaskAttempt(containerId, taskSpec.getTaskAttemptID(), host, port);
- nodesForQuery.add(nodeId);
-
- sourceStateTracker.registerTaskForStateUpdates(host, port, taskSpec.getInputs());
- FragmentRuntimeInfo fragmentRuntimeInfo = sourceStateTracker.getFragmentRuntimeInfo(
- taskSpec.getVertexName(), taskSpec.getTaskAttemptID().getTaskID().getId(), priority);
- SubmitWorkRequestProto requestProto;
-
- try {
- requestProto = constructSubmitWorkRequest(containerId, taskSpec, fragmentRuntimeInfo);
- } catch (IOException e) {
- throw new RuntimeException("Failed to construct request", e);
- }
-
- // Have to register this up front right now. Otherwise, it's possible for the task to start
- // sending out status/DONE/KILLED/FAILED messages before TAImpl knows how to handle them.
- getContext()
- .taskStartedRemotely(taskSpec.getTaskAttemptID(), containerId);
- communicator.sendSubmitWork(requestProto, host, port,
- new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() {
- @Override
- public void setResponse(SubmitWorkResponseProto response) {
- if (response.hasSubmissionState()) {
- LlapDaemonProtocolProtos.SubmissionStateProto ss = response.getSubmissionState();
- if (ss.equals(LlapDaemonProtocolProtos.SubmissionStateProto.REJECTED)) {
- LOG.info(
- "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
- containerId + ", Service Busy");
- getContext().taskKilled(taskSpec.getTaskAttemptID(),
- TaskAttemptEndReason.EXECUTOR_BUSY, "Service Busy");
- return;
- }
- } else {
- // TODO: Provide support for reporting errors
- // This should never happen as server always returns a valid status on success
- throw new RuntimeException("SubmissionState in response is expected!");
- }
- LOG.info("Successfully launched task: " + taskSpec.getTaskAttemptID());
- }
-
- @Override
- public void indicateError(Throwable t) {
- if (t instanceof ServiceException) {
- ServiceException se = (ServiceException) t;
- t = se.getCause();
- }
- if (t instanceof RemoteException) {
- RemoteException re = (RemoteException) t;
- // All others from the remote service cause the task to FAIL.
- LOG.info(
- "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
- containerId, t);
- getContext()
- .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
- t.toString());
- } else {
- // Exception from the RPC layer - communication failure, consider as KILLED / service down.
- if (t instanceof IOException) {
- LOG.info(
- "Unable to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
- containerId + ", Communication Error");
- getContext().taskKilled(taskSpec.getTaskAttemptID(),
- TaskAttemptEndReason.COMMUNICATION_ERROR, "Communication Error");
- } else {
- // Anything else is a FAIL.
- LOG.info(
- "Failed to run task: " + taskSpec.getTaskAttemptID() + " on containerId: " +
- containerId, t);
- getContext()
- .taskFailed(taskSpec.getTaskAttemptID(), TaskAttemptEndReason.OTHER,
- t.getMessage());
- }
- }
- }
- });
- }
-
- @Override
- public void unregisterRunningTaskAttempt(final TezTaskAttemptID taskAttemptId,
- TaskAttemptEndReason endReason,
- String diagnostics) {
- super.unregisterRunningTaskAttempt(taskAttemptId, endReason, diagnostics);
-
- if (endReason == TaskAttemptEndReason.INTERNAL_PREEMPTION) {
- LOG.info("Processing taskEnd for task {} caused by internal preemption", taskAttemptId);
- sendTaskTerminated(taskAttemptId, false);
- }
- entityTracker.unregisterTaskAttempt(taskAttemptId);
- // This will also be invoked for tasks which have been KILLED / rejected by the daemon.
- // Informing the daemon becomes necessary once the LlapScheduler supports preemption
- // and/or starts attempting to kill tasks which may be running on a node.
- }
-
- private void sendTaskTerminated(final TezTaskAttemptID taskAttemptId,
- boolean invokedByContainerEnd) {
- LOG.info(
- "DBG: Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}",
- taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd");
- LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId);
- // NodeId can be null if the task gets unregistered due to failure / being killed by the daemon itself
- if (nodeId != null) {
- TerminateFragmentRequestProto request =
- TerminateFragmentRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifierProto)
- .setFragmentIdentifierString(taskAttemptId.toString()).build();
- communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(),
- new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() {
- @Override
- public void setResponse(TerminateFragmentResponseProto response) {
- }
-
- @Override
- public void indicateError(Throwable t) {
- LOG.warn("Failed to send terminate fragment request for {}",
- taskAttemptId.toString());
- }
- });
- } else {
- LOG.info(
- "Not sending terminate request for fragment {} since it's node is not known. Already unregistered",
- taskAttemptId.toString());
- }
- }
-
-
-
-
- @Override
- public void dagComplete(final int dagIdentifier) {
- QueryCompleteRequestProto request = QueryCompleteRequestProto.newBuilder()
- .setQueryIdentifier(constructQueryIdentifierProto(dagIdentifier))
- .setDeleteDelay(deleteDelayOnDagComplete).build();
- for (final LlapNodeId llapNodeId : nodesForQuery) {
- LOG.info("Sending dagComplete message for {}, to {}", dagIdentifier, llapNodeId);
- communicator.sendQueryComplete(request, llapNodeId.getHostname(), llapNodeId.getPort(),
- new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<LlapDaemonProtocolProtos.QueryCompleteResponseProto>() {
- @Override
- public void setResponse(LlapDaemonProtocolProtos.QueryCompleteResponseProto response) {
- }
-
- @Override
- public void indicateError(Throwable t) {
- LOG.warn("Failed to indicate dag complete dagId={} to node {}", dagIdentifier, llapNodeId);
- }
- });
- }
-
- nodesForQuery.clear();
- // TODO Ideally move some of the other cleanup code from resetCurrentDag over here
- }
-
- @Override
- public void onVertexStateUpdated(VertexStateUpdate vertexStateUpdate) {
- // Delegate updates over to the source state tracker.
- sourceStateTracker
- .sourceStateUpdated(vertexStateUpdate.getVertexName(), vertexStateUpdate.getVertexState());
- }
-
- public void sendStateUpdate(final String host, final int port,
- final SourceStateUpdatedRequestProto request) {
- communicator.sendSourceStateUpdate(request, host, port,
- new LlapDaemonProtocolClientProxy.ExecuteRequestCallback<SourceStateUpdatedResponseProto>() {
- @Override
- public void setResponse(SourceStateUpdatedResponseProto response) {
- }
-
- @Override
- public void indicateError(Throwable t) {
- // TODO HIVE-10280.
- // Ideally, this should be retried for a while, after which the node should be marked as failed.
- // Considering tasks are supposed to run fast. Failing the task immediately may be a good option.
- LOG.error(
- "Failed to send state update to node: " + host + ":" + port + ", StateUpdate=" +
- request, t);
- }
- });
- }
-
-
- private static class PingingNodeInfo {
- final AtomicLong logTimestamp;
- final AtomicInteger pingCount;
-
- PingingNodeInfo(long currentTs) {
- logTimestamp = new AtomicLong(currentTs);
- pingCount = new AtomicInteger(1);
- }
- }
-
- public void registerKnownNode(LlapNodeId nodeId) {
- Long old = knownNodeMap.putIfAbsent(nodeId,
- TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS));
- if (old == null) {
- if (isInfoEnabled) {
- LOG.info("Added new known node: {}", nodeId);
- }
- }
- }
-
- public void registerPingingNode(LlapNodeId nodeId) {
- long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
- PingingNodeInfo ni = new PingingNodeInfo(currentTs);
- PingingNodeInfo old = pingedNodeMap.put(nodeId, ni);
- if (old == null) {
- if (isInfoEnabled) {
- LOG.info("Added new pinging node: [{}]", nodeId);
- }
- } else {
- old.pingCount.incrementAndGet();
- }
- // The node should always be known by this point. Log occasionally if it is not known.
- if (!knownNodeMap.containsKey(nodeId)) {
- if (old == null) {
- // First time this is seen. Log it.
- LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, ni.pingCount.get());
- } else {
- // Pinged before. Log only occasionally.
- if (currentTs > old.logTimestamp.get() + 5000l) { // 5 seconds elapsed. Log again.
- LOG.warn("Received ping from unknownNode: [{}], count={}", nodeId, old.pingCount.get());
- old.logTimestamp.set(currentTs);
- }
- }
-
- }
- }
-
-
- private final AtomicLong nodeNotFoundLogTime = new AtomicLong(0);
-
- void nodePinged(String hostname, int port) {
- LlapNodeId nodeId = LlapNodeId.getInstance(hostname, port);
- registerPingingNode(nodeId);
- BiMap<ContainerId, TezTaskAttemptID> biMap =
- entityTracker.getContainerAttemptMapForNode(nodeId);
- if (biMap != null) {
- synchronized (biMap) {
- for (Map.Entry<ContainerId, TezTaskAttemptID> entry : biMap.entrySet()) {
- getContext().taskAlive(entry.getValue());
- getContext().containerAlive(entry.getKey());
- }
- }
- } else {
- long currentTs = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
- if (currentTs > nodeNotFoundLogTime.get() + 5000l) {
- LOG.warn("Received ping from node without any registered tasks or containers: " + hostname +
- ":" + port +
- ". Could be caused by pre-emption by the AM," +
- " or a mismatched hostname. Enable debug logging for mismatched host names");
- nodeNotFoundLogTime.set(currentTs);
- }
- }
- }
-
- private void resetCurrentDag(int newDagId) {
- // Working on the assumption that a single DAG runs at a time per AM.
- currentQueryIdentifierProto = constructQueryIdentifierProto(newDagId);
- sourceStateTracker.resetState(newDagId);
- nodesForQuery.clear();
- LOG.info("CurrentDagId set to: " + newDagId + ", name=" + getContext().getCurrentDagName());
- // TODO Is it possible for heartbeats to come in from lost tasks - those should be told to die, which
- // is likely already happening.
- }
-
- private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId,
- TaskSpec taskSpec,
- FragmentRuntimeInfo fragmentRuntimeInfo) throws
- IOException {
- SubmitWorkRequestProto.Builder builder =
- SubmitWorkRequestProto.newBuilder(BASE_SUBMIT_WORK_REQUEST);
- builder.setContainerIdString(containerId.toString());
- builder.setAmHost(getAddress().getHostName());
- builder.setAmPort(getAddress().getPort());
- Credentials taskCredentials = new Credentials();
- // Credentials can change across DAGs. Ideally construct only once per DAG.
- taskCredentials.addAll(getContext().getCredentials());
-
- Preconditions.checkState(currentQueryIdentifierProto.getDagIdentifier() ==
- taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId());
- ByteBuffer credentialsBinary = credentialMap.get(currentQueryIdentifierProto);
- if (credentialsBinary == null) {
- credentialsBinary = serializeCredentials(getContext().getCredentials());
- credentialMap.putIfAbsent(currentQueryIdentifierProto, credentialsBinary.duplicate());
- } else {
- credentialsBinary = credentialsBinary.duplicate();
- }
- builder.setCredentialsBinary(ByteString.copyFrom(credentialsBinary));
- builder.setFragmentSpec(Converters.convertTaskSpecToProto(taskSpec));
- builder.setFragmentRuntimeInfo(fragmentRuntimeInfo);
- return builder.build();
- }
-
- private ByteBuffer serializeCredentials(Credentials credentials) throws IOException {
- Credentials containerCredentials = new Credentials();
- containerCredentials.addAll(credentials);
- DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
- containerCredentials.writeTokenStorageToStream(containerTokens_dob);
- return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
- }
-
-
-
- protected class LlapTaskUmbilicalProtocolImpl implements LlapTaskUmbilicalProtocol {
-
- private final TezTaskUmbilicalProtocol tezUmbilical;
-
- public LlapTaskUmbilicalProtocolImpl(TezTaskUmbilicalProtocol tezUmbilical) {
- this.tezUmbilical = tezUmbilical;
- }
-
- @Override
- public boolean canCommit(TezTaskAttemptID taskid) throws IOException {
- return tezUmbilical.canCommit(taskid);
- }
-
- @Override
- public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException,
- TezException {
- return tezUmbilical.heartbeat(request);
- }
-
- @Override
- public void nodeHeartbeat(Text hostname, int port) throws IOException {
- nodePinged(hostname.toString(), port);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received heartbeat from [" + hostname + ":" + port +"]");
- }
- }
-
- @Override
- public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException {
- // TODO Unregister the task for state updates, which could in turn unregister the node.
- getContext().taskKilled(taskAttemptId,
- TaskAttemptEndReason.EXTERNAL_PREEMPTION, "Attempt preempted");
- entityTracker.unregisterTaskAttempt(taskAttemptId);
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
- return versionID;
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
- int clientMethodsHash) throws IOException {
- return ProtocolSignature.getProtocolSignature(this, protocol,
- clientVersion, clientMethodsHash);
- }
- }
-
- /**
- * Track the association between known containers and taskAttempts, along with the nodes they are assigned to.
- */
- @VisibleForTesting
- static final class EntityTracker {
- @VisibleForTesting
- final ConcurrentMap<TezTaskAttemptID, LlapNodeId> attemptToNodeMap = new ConcurrentHashMap<>();
- @VisibleForTesting
- final ConcurrentMap<ContainerId, LlapNodeId> containerToNodeMap = new ConcurrentHashMap<>();
- @VisibleForTesting
- final ConcurrentMap<LlapNodeId, BiMap<ContainerId, TezTaskAttemptID>> nodeMap = new ConcurrentHashMap<>();
-
- void registerTaskAttempt(ContainerId containerId, TezTaskAttemptID taskAttemptId, String host, int port) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Registering " + containerId + ", " + taskAttemptId + " for node: " + host + ":" + port);
- }
- LlapNodeId llapNodeId = LlapNodeId.getInstance(host, port);
- attemptToNodeMap.putIfAbsent(taskAttemptId, llapNodeId);
-
- registerContainer(containerId, host, port);
-
- // nodeMap registration.
- BiMap<ContainerId, TezTaskAttemptID> tmpMap = HashBiMap.create();
- BiMap<ContainerId, TezTaskAttemptID> old = nodeMap.putIfAbsent(llapNodeId, tmpMap);
- BiMap<ContainerId, TezTaskAttemptID> usedInstance;
- usedInstance = old == null ? tmpMap : old;
- synchronized(usedInstance) {
- usedInstance.put(containerId, taskAttemptId);
- }
- // Make sure to put the instance back again, in case it was removed as part of a
- // containerEnd/taskEnd invocation.
- nodeMap.putIfAbsent(llapNodeId, usedInstance);
- }
-
- void unregisterTaskAttempt(TezTaskAttemptID attemptId) {
- LlapNodeId llapNodeId = attemptToNodeMap.remove(attemptId);
- if (llapNodeId == null) {
- // Possible since either container / task can be unregistered.
- return;
- }
-
- BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
- ContainerId matched = null;
- if (bMap != null) {
- synchronized(bMap) {
- matched = bMap.inverse().remove(attemptId);
- }
- if (bMap.isEmpty()) {
- nodeMap.remove(llapNodeId);
- }
- }
-
- // Remove the container mapping
- if (matched != null) {
- containerToNodeMap.remove(matched);
- }
-
- }
-
- void registerContainer(ContainerId containerId, String hostname, int port) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Registering " + containerId + " for node: " + hostname + ":" + port);
- }
- containerToNodeMap.putIfAbsent(containerId, LlapNodeId.getInstance(hostname, port));
- // nodeMap registration is not required, since there's no taskId association.
- }
-
- LlapNodeId getNodeIdForContainer(ContainerId containerId) {
- return containerToNodeMap.get(containerId);
- }
-
- LlapNodeId getNodeIdForTaskAttempt(TezTaskAttemptID taskAttemptId) {
- return attemptToNodeMap.get(taskAttemptId);
- }
-
- ContainerId getContainerIdForAttempt(TezTaskAttemptID taskAttemptId) {
- LlapNodeId llapNodeId = getNodeIdForTaskAttempt(taskAttemptId);
- if (llapNodeId != null) {
- BiMap<TezTaskAttemptID, ContainerId> bMap = nodeMap.get(llapNodeId).inverse();
- if (bMap != null) {
- synchronized (bMap) {
- return bMap.get(taskAttemptId);
- }
- } else {
- return null;
- }
- } else {
- return null;
- }
- }
-
- TezTaskAttemptID getTaskAttemptIdForContainer(ContainerId containerId) {
- LlapNodeId llapNodeId = getNodeIdForContainer(containerId);
- if (llapNodeId != null) {
- BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
- if (bMap != null) {
- synchronized (bMap) {
- return bMap.get(containerId);
- }
- } else {
- return null;
- }
- } else {
- return null;
- }
- }
-
- void unregisterContainer(ContainerId containerId) {
- LlapNodeId llapNodeId = containerToNodeMap.remove(containerId);
- if (llapNodeId == null) {
- // Possible since either container / task can be unregistered.
- return;
- }
-
- BiMap<ContainerId, TezTaskAttemptID> bMap = nodeMap.get(llapNodeId);
- TezTaskAttemptID matched = null;
- if (bMap != null) {
- synchronized(bMap) {
- matched = bMap.remove(containerId);
- }
- if (bMap.isEmpty()) {
- nodeMap.remove(llapNodeId);
- }
- }
-
- // Remove the container mapping
- if (matched != null) {
- attemptToNodeMap.remove(matched);
- }
- }
-
- /**
- * Return a {@link BiMap} containing container->taskAttemptId mapping for the host specified.
- * </p>
- * <p/>
- * This method return the internal structure used by the EntityTracker. Users must synchronize
- * on the structure to ensure correct usage.
- *
- * @param llapNodeId
- * @return
- */
- BiMap<ContainerId, TezTaskAttemptID> getContainerAttemptMapForNode(LlapNodeId llapNodeId) {
- BiMap<ContainerId, TezTaskAttemptID> biMap = nodeMap.get(llapNodeId);
- return biMap;
- }
-
- }
-
- private QueryIdentifierProto constructQueryIdentifierProto(int dagIdentifier) {
- return QueryIdentifierProto.newBuilder()
- .setAppIdentifier(getContext().getCurrentAppIdentifier()).setDagIdentifier(dagIdentifier)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
deleted file mode 100644
index 4102f5b..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapUmbilicalPolicyProvider.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol;
-import org.apache.hadoop.security.authorize.Service;
-import org.apache.tez.dag.api.TezConstants;
-import org.apache.tez.dag.app.security.authorize.TezAMPolicyProvider;
-
-public class LlapUmbilicalPolicyProvider extends TezAMPolicyProvider {
-
- private static Service[] services;
- private static final Object servicesLock = new Object();
-
- @Override
- public Service[] getServices() {
- if (services != null) return services;
- synchronized (servicesLock) {
- if (services != null) return services;
- Service[] parentSvc = super.getServices();
- Service[] result = Arrays.copyOf(parentSvc, parentSvc.length + 1);
- result[parentSvc.length] = new Service(
- TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL,
- LlapTaskUmbilicalProtocol.class);
- return (services = result);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java b/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
deleted file mode 100644
index fded9bf..0000000
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/tezplugins/helpers/SourceStateTracker.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.llap.tezplugins.helpers;
-
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto;
-import org.apache.tez.serviceplugins.api.TaskCommunicatorContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.llap.LlapNodeId;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.FragmentRuntimeInfo;
-import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
-import org.apache.hadoop.hive.llap.tezplugins.Converters;
-import org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator;
-import org.apache.tez.dag.api.event.VertexState;
-import org.apache.tez.mapreduce.input.MRInput;
-import org.apache.tez.mapreduce.input.MRInputLegacy;
-import org.apache.tez.mapreduce.input.MultiMRInput;
-import org.apache.tez.runtime.api.impl.InputSpec;
-
-public class SourceStateTracker {
-
- private static final Logger LOG = LoggerFactory.getLogger(SourceStateTracker.class);
-
- private final TaskCommunicatorContext taskCommunicatorContext;
- private final LlapTaskCommunicator taskCommunicator;
-
- private final QueryIdentifierProto BASE_QUERY_IDENTIFIER;
-
- // Tracks vertices for which notifications have been registered
- private final Set<String> notificationRegisteredVertices = new HashSet<>();
-
- private final Map<String, SourceInfo> sourceInfoMap = new HashMap<>();
- private final Map<LlapNodeId, NodeInfo> nodeInfoMap = new HashMap<>();
-
- private volatile QueryIdentifierProto currentQueryIdentifier;
-
- public SourceStateTracker(TaskCommunicatorContext taskCommunicatorContext,
- LlapTaskCommunicator taskCommunicator) {
- this.taskCommunicatorContext = taskCommunicatorContext;
- this.taskCommunicator = taskCommunicator;
- BASE_QUERY_IDENTIFIER = QueryIdentifierProto.newBuilder()
- .setAppIdentifier(taskCommunicatorContext.getCurrentAppIdentifier()).build();
- }
-
- /**
- * To be invoked after each DAG completes.
- */
- public synchronized void resetState(int newDagId) {
- sourceInfoMap.clear();
- nodeInfoMap.clear();
- notificationRegisteredVertices.clear();
- this.currentQueryIdentifier =
- QueryIdentifierProto.newBuilder(BASE_QUERY_IDENTIFIER).setDagIdentifier(newDagId).build();
- }
-
- /**
- * Used to register a task for state updates. Effectively registers for state updates to go to the specific node.
- * @param host
- * @param port
- * @param inputSpecList
- */
- public synchronized void registerTaskForStateUpdates(String host, int port,
- List<InputSpec> inputSpecList) {
-
- // Add tracking information. Check if source state already known and send out an update if it is.
-
- List<String> sourcesOfInterest = getSourceInterestList(inputSpecList);
- if (sourcesOfInterest != null && !sourcesOfInterest.isEmpty()) {
- LlapNodeId nodeId = LlapNodeId.getInstance(host, port);
- NodeInfo nodeInfo = getNodeInfo(nodeId);
-
- // Set up the data structures, before any notifications come in.
- for (String src : sourcesOfInterest) {
- VertexState oldStateForNode = nodeInfo.getLastKnownStateForSource(src);
- if (oldStateForNode == null) {
- // Not registered for this node.
- // Register and send state if it is successful.
- SourceInfo srcInfo = getSourceInfo(src);
- srcInfo.addNode(nodeId);
-
- nodeInfo.addSource(src, srcInfo.lastKnownState);
- if (srcInfo.lastKnownState == VertexState.SUCCEEDED) {
- sendStateUpdateToNode(nodeId, src, srcInfo.lastKnownState);
- }
-
- } else {
- // Already registered to send updates to this node for the specific source.
- // Nothing to do for now, unless tracking tasks at a later point.
- }
-
- // Setup for actual notifications, if not already done for a previous task.
- maybeRegisterForVertexUpdates(src);
- }
- } else {
- // Don't need to track anything for this task. No new notifications, etc.
- }
- }
-
- /**
- * Handled notifications on state updates for sources
- * @param sourceName
- * @param sourceState
- */
- public synchronized void sourceStateUpdated(String sourceName, VertexState sourceState) {
- SourceInfo sourceInfo = getSourceInfo(sourceName);
- // Update source info if the state is SUCCEEDED
- if (sourceState == VertexState.SUCCEEDED) {
- sourceInfo.numCompletedTasks = taskCommunicatorContext.getVertexCompletedTaskCount(sourceName);
- sourceInfo.numTasks = taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
- }
- sourceInfo.lastKnownState = sourceState;
- // Checking state per node for future failure handling scenarios, where an update
- // to a single node may fail.
- for (LlapNodeId nodeId : sourceInfo.getInterestedNodes()) {
- NodeInfo nodeInfo = nodeInfoMap.get(nodeId);
- VertexState lastStateForNode = nodeInfo.getLastKnownStateForSource(sourceName);
- // Send only if the state has changed.
- if (lastStateForNode != sourceState) {
- nodeInfo.setLastKnownStateForSource(sourceName, sourceState);
- sendStateUpdateToNode(nodeId, sourceName, sourceState);
- }
- }
- }
-
-
- // Assumes serialized DAGs within an AM, and a reset of structures after each DAG completes.
- /**
- * Constructs FragmentRuntimeInfo for scheduling within LLAP daemons.
- * Also caches state based on state updates.
- * @param vertexName
- * @param fragmentNumber
- * @param priority
- * @return
- */
- public synchronized FragmentRuntimeInfo getFragmentRuntimeInfo(String vertexName, int fragmentNumber,
- int priority) {
- FragmentRuntimeInfo.Builder builder = FragmentRuntimeInfo.newBuilder();
- maybeRegisterForVertexUpdates(vertexName);
-
- MutableInt totalTaskCount = new MutableInt(0);
- MutableInt completedTaskCount = new MutableInt(0);
- computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, vertexName);
-
- builder.setNumSelfAndUpstreamCompletedTasks(completedTaskCount.intValue());
- builder.setNumSelfAndUpstreamTasks(totalTaskCount.intValue());
- builder.setDagStartTime(taskCommunicatorContext.getDagStartTime());
- builder.setWithinDagPriority(priority);
- builder.setFirstAttemptStartTime(taskCommunicatorContext.getFirstAttemptStartTime(vertexName, fragmentNumber));
- builder.setCurrentAttemptStartTime(System.currentTimeMillis());
- return builder.build();
- }
-
- private void computeUpstreamTaskCounts(MutableInt completedTaskCount, MutableInt totalTaskCount, String sourceName) {
- SourceInfo sourceInfo = getSourceInfo(sourceName);
- if (sourceInfo.lastKnownState == VertexState.SUCCEEDED) {
- // Some of the information in the source is complete. Don't need to fetch it from the context.
- completedTaskCount.add(sourceInfo.numCompletedTasks);
- totalTaskCount.add(sourceInfo.numTasks);
- } else {
- completedTaskCount.add(taskCommunicatorContext.getVertexCompletedTaskCount(sourceName));
- int totalCount =taskCommunicatorContext.getVertexTotalTaskCount(sourceName);
- // Uninitialized vertices will report count as 0.
- totalCount = totalCount == -1 ? 0 : totalCount;
- totalTaskCount.add(totalCount);
- }
-
- // Walk through all the source vertices
- for (String up : taskCommunicatorContext.getInputVertexNames(sourceName)) {
- computeUpstreamTaskCounts(completedTaskCount, totalTaskCount, up);
- }
- }
-
- private static class SourceInfo {
-
- // Always start in the running state. Requests for state updates will be sent out after registration.
- private VertexState lastKnownState = VertexState.RUNNING;
-
- // Used for sending notifications about a vertex completed. For canFinish
- // Can be converted to a Tez event, if this is sufficient to decide on pre-emption
- private final List<LlapNodeId> interestedNodes = new LinkedList<>();
-
- // Used for sending information for scheduling priority.
- private int numTasks;
- private int numCompletedTasks;
-
- void addNode(LlapNodeId nodeId) {
- interestedNodes.add(nodeId);
- }
-
- List<LlapNodeId> getInterestedNodes() {
- return this.interestedNodes;
- }
-
-
-
- }
-
- private synchronized SourceInfo getSourceInfo(String srcName) {
- SourceInfo sourceInfo = sourceInfoMap.get(srcName);
- if (sourceInfo == null) {
- sourceInfo = new SourceInfo();
- sourceInfoMap.put(srcName, sourceInfo);
- }
- return sourceInfo;
- }
-
-
- private static class NodeInfo {
- private final Map<String, VertexState> sourcesOfInterest = new HashMap<>();
-
- void addSource(String srcName, VertexState sourceState) {
- sourcesOfInterest.put(srcName, sourceState);
- }
-
- VertexState getLastKnownStateForSource(String src) {
- return sourcesOfInterest.get(src);
- }
-
- void setLastKnownStateForSource(String src, VertexState state) {
- sourcesOfInterest.put(src, state);
- }
- }
-
- private synchronized NodeInfo getNodeInfo(LlapNodeId llapNodeId) {
- NodeInfo nodeInfo = nodeInfoMap.get(llapNodeId);
- if (nodeInfo == null) {
- nodeInfo = new NodeInfo();
- nodeInfoMap.put(llapNodeId, nodeInfo);
- }
- return nodeInfo;
- }
-
-
- private List<String> getSourceInterestList(List<InputSpec> inputSpecList) {
- List<String> sourcesOfInterest = Collections.emptyList();
- if (inputSpecList != null) {
- boolean alreadyFound = false;
- for (InputSpec inputSpec : inputSpecList) {
- if (isSourceOfInterest(inputSpec)) {
- if (!alreadyFound) {
- alreadyFound = true;
- sourcesOfInterest = new LinkedList<>();
- }
- sourcesOfInterest.add(inputSpec.getSourceVertexName());
- }
- }
- }
- return sourcesOfInterest;
- }
-
-
- private void maybeRegisterForVertexUpdates(String sourceName) {
- if (!notificationRegisteredVertices.contains(sourceName)) {
- notificationRegisteredVertices.add(sourceName);
- taskCommunicatorContext.registerForVertexStateUpdates(sourceName, EnumSet.of(
- VertexState.RUNNING, VertexState.SUCCEEDED));
- }
- }
-
- private boolean isSourceOfInterest(InputSpec inputSpec) {
- String inputClassName = inputSpec.getInputDescriptor().getClassName();
- // MRInput is not of interest since it'll always be ready.
- return !(inputClassName.equals(MRInputLegacy.class.getName()) || inputClassName.equals(
- MultiMRInput.class.getName()) || inputClassName.equals(MRInput.class.getName()));
- }
-
- void sendStateUpdateToNode(LlapNodeId nodeId, String sourceName, VertexState state) {
- taskCommunicator.sendStateUpdate(nodeId.getHostname(), nodeId.getPort(),
- SourceStateUpdatedRequestProto.newBuilder().setQueryIdentifier(currentQueryIdentifier)
- .setSrcName(sourceName).setState(Converters.fromVertexState(state)).build());
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9886414b/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java b/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java
deleted file mode 100644
index 89b7198..0000000
--- a/llap-server/src/java/org/apache/tez/dag/app/rm/ContainerFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.app.rm;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-class ContainerFactory {
- final ApplicationAttemptId customAppAttemptId;
- AtomicLong nextId;
-
- public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) {
- this.nextId = new AtomicLong(1);
- ApplicationId appId =
- ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId());
- this.customAppAttemptId =
- ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId());
- }
-
- public Container createContainer(Resource capability, Priority priority, String hostname,
- int port) {
- ContainerId containerId =
- ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement());
- NodeId nodeId = NodeId.newInstance(hostname, port);
- String nodeHttpAddress = "hostname:0"; // TODO: include UI ports
-
- Container container =
- Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null);
-
- return container;
- }
-}